From d41320d4efec7e3b23608d63ddb0e32272d86583 Mon Sep 17 00:00:00 2001 From: Alex North <445306+anorth@users.noreply.github.com> Date: Fri, 27 Oct 2023 14:35:04 +1300 Subject: [PATCH] Add ListAllocations/Claims methods for verified registry actor. --- Cargo.lock | 45 ++++-- Cargo.toml | 2 +- actors/verifreg/Cargo.toml | 1 + actors/verifreg/src/expiration.rs | 3 +- actors/verifreg/src/lib.rs | 29 +++- actors/verifreg/src/state.rs | 104 ++++++++++++- actors/verifreg/src/types.rs | 37 +++++ actors/verifreg/tests/harness/mod.rs | 47 +++++- actors/verifreg/tests/verifreg_actor_test.rs | 152 ++++++++++++++++++- runtime/src/lib.rs | 7 +- runtime/src/util/mapmap.rs | 87 ++++++++++- 11 files changed, 481 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd6df0910a..f4f9f7a4a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1425,7 +1425,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "lazy_static", "log", @@ -1509,7 +1509,7 @@ dependencies = [ "frc42_dispatch", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "log", "num-derive 0.3.3", @@ -1533,7 +1533,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "integer-encoding 3.0.4", "itertools 0.10.5", @@ -1565,7 +1565,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "itertools 0.10.5", "lazy_static", @@ -1589,7 +1589,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "indexmap 1.9.3", "integer-encoding 3.0.4", @@ -1633,7 +1633,7 @@ dependencies = [ "frc42_dispatch", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "indexmap 1.9.3", "integer-encoding 3.0.4", @@ -1685,9 +1685,10 @@ dependencies = [ "frc42_dispatch", "frc46_token", "fvm_actor_utils", + "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "lazy_static", "log", @@ -1743,7 +1744,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "hex", "hex-literal", @@ -1780,7 +1781,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_sdk", "fvm_shared", "hex", @@ -1948,7 +1949,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.8.0", "fvm_sdk", "fvm_shared", "integer-encoding 4.0.0", @@ -2198,6 +2199,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "fvm_ipld_hamt" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c900736087ff87cc51f669eee2f8e000c80717472242eb3f712aaa059ac3b3" +dependencies = [ + "anyhow", + "byteorder", + "cid 0.10.1", + "forest_hash_utils", + "fvm_ipld_blockstore", + "fvm_ipld_encoding", + "libipld-core 0.16.0", + "multihash 0.18.1", + "once_cell", + "serde", + "sha2 0.10.7", + "thiserror", +] + [[package]] name = "fvm_ipld_kamt" version = "0.3.0" @@ -4087,7 +4108,7 @@ dependencies = [ "fil_builtin_actors_state", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "integer-encoding 3.0.4", "multihash 0.18.1", @@ -4403,7 +4424,7 @@ dependencies = [ "cid 0.10.1", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "num-derive 0.3.3", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index 15f32ac590..9f7aaa1514 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,7 +118,7 @@ fvm_sdk = "~4.0" fvm_shared = "~4.0" fvm_ipld_encoding = "0.4.0" fvm_ipld_blockstore = "0.2.0" -fvm_ipld_hamt = "0.8.0" +fvm_ipld_hamt = "0.9.0" fvm_ipld_kamt = "0.3.0" fvm_ipld_amt = { version = "0.6.2" } fvm_ipld_bitfield = "0.6.0" diff --git a/actors/verifreg/Cargo.toml b/actors/verifreg/Cargo.toml index 01e81459b0..6767f1c8d2 100644 --- a/actors/verifreg/Cargo.toml +++ b/actors/verifreg/Cargo.toml @@ -21,6 +21,7 @@ cid = { workspace = true } frc42_dispatch = { workspace = true } frc46_token = { workspace = true } fvm_actor_utils = { workspace = true } +fvm_ipld_bitfield = { workspace = true } fvm_ipld_blockstore = { workspace = true } fvm_ipld_encoding = { workspace = true } fvm_ipld_hamt = { workspace = true } diff --git a/actors/verifreg/src/expiration.rs b/actors/verifreg/src/expiration.rs index 9df270e692..12f19798ac 100644 --- a/actors/verifreg/src/expiration.rs +++ b/actors/verifreg/src/expiration.rs @@ -42,8 +42,7 @@ where collection .for_each_in(owner, |key, record| { if curr_epoch >= record.expiration() { - let id = parse_uint_key(key) - .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to parse uint key")?; + let id = parse_uint_key(key)?; found_ids.push(id); } Ok(()) diff --git a/actors/verifreg/src/lib.rs b/actors/verifreg/src/lib.rs index 18d5f617f8..f2671d1225 100644 --- a/actors/verifreg/src/lib.rs +++ b/actors/verifreg/src/lib.rs @@ -31,7 +31,8 @@ use fil_actors_runtime::{ActorContext, AsActorError, BatchReturnGen}; use crate::ext::datacap::{DestroyParams, MintParams}; use crate::state::{ - DataCapMap, RemoveDataCapProposalMap, DATACAP_MAP_CONFIG, REMOVE_DATACAP_PROPOSALS_CONFIG, + Cursor, DataCapMap, RemoveDataCapProposalMap, DATACAP_MAP_CONFIG, + REMOVE_DATACAP_PROPOSALS_CONFIG, }; pub use self::state::Allocation; @@ -70,6 +71,8 @@ pub enum Method { GetClaimsExported = frc42_dispatch::method_hash!("GetClaims"), ExtendClaimTermsExported = frc42_dispatch::method_hash!("ExtendClaimTerms"), RemoveExpiredClaimsExported = frc42_dispatch::method_hash!("RemoveExpiredClaims"), + ListAllocationsExported = frc42_dispatch::method_hash!("ListAllocations"), + ListClaimsExported = frc42_dispatch::method_hash!("ListClaims"), UniversalReceiverHook = frc42_dispatch::method_hash!("Receive"), } @@ -604,6 +607,28 @@ impl Actor { Ok(RemoveExpiredClaimsReturn { considered, results: batch_ret }) } + pub fn list_allocations( + rt: &impl Runtime, + params: ListAllocationsParams, + ) -> Result { + let cursor = Cursor::from_bytes(params.cursor)?; + let st: State = rt.state()?; + let (allocations, next_cursor) = st.list_allocations(rt.store(), cursor, Some(params.limit))?; + let next_cursor = next_cursor.map(|c| c.to_bytes()).transpose()?; + Ok(ListAllocationsResponse { allocations, next_cursor }) + } + + pub fn list_claims( + rt: &impl Runtime, + params: ListClaimsParams, + ) -> Result { + let cursor = Cursor::from_bytes(params.cursor)?; + let st: State = rt.state()?; + let (claims, next_cursor) = st.list_claims(rt.store(), cursor, Some(params.limit))?; + let next_cursor = next_cursor.map(|c| c.to_bytes()).transpose()?; + Ok(ListClaimsResponse { claims, next_cursor }) + } + // Receives data cap tokens (only) and creates allocations according to one or more // allocation requests specified in the transfer's operator data. // The token amount received must exactly correspond to the sum of the requested allocation sizes. @@ -1069,6 +1094,8 @@ impl ActorCode for Actor { GetClaims|GetClaimsExported => get_claims, ExtendClaimTerms|ExtendClaimTermsExported => extend_claim_terms, RemoveExpiredClaims|RemoveExpiredClaimsExported => remove_expired_claims, + ListAllocationsExported => list_allocations, + ListClaimsExported => list_claims, UniversalReceiverHook => universal_receiver_hook, } } diff --git a/actors/verifreg/src/state.rs b/actors/verifreg/src/state.rs index 10c983f8fa..ee33e441fc 100644 --- a/actors/verifreg/src/state.rs +++ b/actors/verifreg/src/state.rs @@ -3,6 +3,7 @@ use cid::Cid; use fvm_ipld_blockstore::Blockstore; +use fvm_ipld_encoding::RawBytes; use fvm_ipld_encoding::tuple::*; use fvm_shared::address::Address; use fvm_shared::bigint::bigint_ser::BigIntDe; @@ -13,10 +14,11 @@ use fvm_shared::sector::SectorNumber; use fvm_shared::{ActorID, HAMT_BIT_WIDTH}; use fil_actors_runtime::{ - actor_error, ActorError, AsActorError, Config, Map2, MapMap, DEFAULT_HAMT_CONFIG, + actor_error, parse_uint_key, ActorError, AsActorError, Config, Map2, MapMap, + DEFAULT_HAMT_CONFIG, }; -use crate::{AddrPairKey, AllocationID, ClaimID}; +use crate::{AddrPairKey, AllocationID, AllocationKey, ClaimID, ClaimKey}; use crate::{DataCap, RemoveDataCapProposalID}; pub type DataCapMap = Map2; @@ -156,6 +158,34 @@ impl State { Ok(allocated_ids) } + // Lists all allocation clients and IDs, paginated with a cursor. + // Returns a new cursor from which to continue listing, if any items remain. + pub fn list_allocations( + &self, + store: &BS, + cursor: Option, + limit: Option, + ) -> Result<(Vec, Option), ActorError> { + let (start_outer, start_inner) = verify_cursor(&cursor, self.allocations)?; + let mut allocs = self.load_allocs(store)?; + let mut found = vec![]; + let next = allocs + .for_each_each(start_outer, start_inner, limit, |k1, k2, _v| { + let client = parse_uint_key(k1)?; + let id = parse_uint_key(k2)?; + found.push(AllocationKey { client, id }); + Ok(()) + }) + .context_code(ExitCode::USR_ILLEGAL_STATE, "listing allocations")?; + let next_cursor = match next { + Some((k1, k2)) => { + Some(Cursor::new(self.allocations, parse_uint_key(&k1)?, parse_uint_key(&k2)?)) + } + None => None, + }; + Ok((found, next_cursor)) + } + pub fn load_claims<'a, BS: Blockstore>( &self, store: &'a BS, @@ -196,7 +226,36 @@ impl State { self.save_claims(&mut st_claims)?; Ok(()) } + + // Lists all claim providers and IDs, paginated with a cursor. + // Returns a new cursor from which to continue listing, if any items remain. + pub fn list_claims( + &self, + store: &BS, + cursor: Option, + limit: Option, + ) -> Result<(Vec, Option), ActorError> { + let (start_outer, start_inner) = verify_cursor(&cursor, self.claims)?; + let mut claims = self.load_claims(store)?; + let mut found = vec![]; + let next = claims + .for_each_each(start_outer, start_inner, limit, |k1, k2, _v| { + let provider = parse_uint_key(k1)?; + let id = parse_uint_key(k2)?; + found.push(ClaimKey { provider, id }); + Ok(()) + }) + .context_code(ExitCode::USR_ILLEGAL_STATE, "listing claims")?; + let next_cursor = match next { + Some((k1, k2)) => { + Some(Cursor::new(self.claims, parse_uint_key(&k1)?, parse_uint_key(&k2)?)) + } + None => None, + }; + Ok((found, next_cursor)) + } } + #[derive(Serialize_tuple, Deserialize_tuple, Clone, Debug, PartialEq, Eq)] pub struct Claim { // The provider storing the data (from allocation). @@ -262,3 +321,44 @@ where .get(provider, id) .context_code(ExitCode::USR_ILLEGAL_STATE, "HAMT lookup failure getting claim") } + +/// Opaque cursor to iterate over allocations/claims data structures +#[derive(Serialize_tuple, Deserialize_tuple, Clone, Debug)] +pub struct Cursor { + pub root: Cid, + pub outer_key: ActorID, + pub inner_key: u64, +} + +impl Cursor { + pub fn new(cid: Cid, outer_key: ActorID, inner_key: u64) -> Self { + Self { root: cid, inner_key, outer_key } + } + + /// Generates a cursor from an opaque representation + pub fn from_bytes(bytes: RawBytes) -> Result, ActorError> { + if bytes.is_empty() { + Ok(None) + } else { + Ok(Some(fvm_ipld_encoding::from_slice(&bytes)?)) + } + } + + /// Generates an opaque representation of the cursor that can be used to resume enumeration + pub fn to_bytes(&self) -> Result { + Ok(RawBytes::from(fvm_ipld_encoding::to_vec(self)?)) + } +} + +fn verify_cursor( + cursor: &Option, + expected_root: Cid, +) -> Result<(Option<&ActorID>, Option<&u64>), ActorError> { + if let Some(cursor) = cursor { + if cursor.root != expected_root { + return Err(ActorError::illegal_argument("invalid cursor".to_string())); + } + return Ok((Some(&cursor.outer_key), Some(&cursor.inner_key))) + } + Ok((None, None)) +} diff --git a/actors/verifreg/src/types.rs b/actors/verifreg/src/types.rs index a58eb53104..c278edc863 100644 --- a/actors/verifreg/src/types.rs +++ b/actors/verifreg/src/types.rs @@ -4,6 +4,7 @@ use cid::Cid; use fil_actors_runtime::{BatchReturn, MapKey}; use fvm_ipld_encoding::tuple::*; +use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; use fvm_shared::bigint::{bigint_ser, BigInt}; use fvm_shared::clock::ChainEpoch; @@ -258,3 +259,39 @@ pub struct RemoveExpiredClaimsReturn { // Results for each processed claim. pub results: BatchReturn, } + +#[derive(Clone, Debug, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)] +pub struct ListAllocationsParams { + pub cursor: RawBytes, + pub limit: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize_tuple, Deserialize_tuple)] +pub struct AllocationKey { + pub client: ActorID, + pub id: AllocationID, +} + +#[derive(Clone, Debug, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct ListAllocationsResponse { + pub allocations: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)] +pub struct ListClaimsParams { + pub cursor: RawBytes, + pub limit: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize_tuple, Deserialize_tuple)] +pub struct ClaimKey { + pub provider: ActorID, + pub id: ClaimID, +} + +#[derive(Clone, Debug, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct ListClaimsResponse { + pub claims: Vec, + pub next_cursor: Option, +} diff --git a/actors/verifreg/tests/harness/mod.rs b/actors/verifreg/tests/harness/mod.rs index c5f2373054..9b9771f2ce 100644 --- a/actors/verifreg/tests/harness/mod.rs +++ b/actors/verifreg/tests/harness/mod.rs @@ -19,14 +19,7 @@ use fvm_shared::{ActorID, MethodNum, HAMT_BIT_WIDTH}; use num_traits::{ToPrimitive, Zero}; use fil_actor_verifreg::testing::check_state_invariants; -use fil_actor_verifreg::{ - ext, Actor as VerifregActor, AddVerifiedClientParams, AddVerifierParams, Allocation, - AllocationClaim, AllocationID, AllocationRequest, AllocationRequests, AllocationsResponse, - Claim, ClaimAllocationsParams, ClaimAllocationsReturn, ClaimExtensionRequest, ClaimID, DataCap, - ExtendClaimTermsParams, ExtendClaimTermsReturn, GetClaimsParams, GetClaimsReturn, Method, - RemoveExpiredAllocationsParams, RemoveExpiredAllocationsReturn, RemoveExpiredClaimsParams, - RemoveExpiredClaimsReturn, SectorAllocationClaims, State, -}; +use fil_actor_verifreg::{ext, Actor as VerifregActor, AddVerifiedClientParams, AddVerifierParams, Allocation, AllocationClaim, AllocationID, AllocationKey, AllocationRequest, AllocationRequests, AllocationsResponse, Claim, ClaimAllocationsParams, ClaimAllocationsReturn, ClaimExtensionRequest, ClaimID, DataCap, ExtendClaimTermsParams, ExtendClaimTermsReturn, GetClaimsParams, GetClaimsReturn, ListAllocationsParams, ListAllocationsResponse, Method, RemoveExpiredAllocationsParams, RemoveExpiredAllocationsReturn, RemoveExpiredClaimsParams, RemoveExpiredClaimsReturn, SectorAllocationClaims, State, ClaimKey, ListClaimsParams, ListClaimsResponse}; use fil_actors_runtime::cbor::serialize; use fil_actors_runtime::runtime::builtins::Type; use fil_actors_runtime::runtime::policy_constants::{ @@ -257,6 +250,25 @@ impl Harness { allocs.get(client, id).unwrap().cloned() } + pub fn list_allocs( + &self, + rt: &MockRuntime, + cursor: &RawBytes, + limit: u64, + ) -> (Vec, Option) { + let params = ListAllocationsParams { cursor: cursor.clone(), limit }; + let ret: ListAllocationsResponse = rt + .call::( + Method::ListAllocationsExported as MethodNum, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap() + .unwrap() + .deserialize() + .expect("failed to deserialize list allocations return"); + (ret.allocations, ret.next_cursor) + } + // Invokes the ClaimAllocations actor method pub fn claim_allocations( &self, @@ -355,6 +367,25 @@ impl Harness { Ok(ret) } + pub fn list_claims( + &self, + rt: &MockRuntime, + cursor: &RawBytes, + limit: u64, + ) -> (Vec, Option) { + let params = ListClaimsParams { cursor: cursor.clone(), limit }; + let ret: ListClaimsResponse = rt + .call::( + Method::ListClaimsExported as MethodNum, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap() + .unwrap() + .deserialize() + .expect("failed to deserialize list claims return"); + (ret.claims, ret.next_cursor) + } + pub fn load_claim(&self, rt: &MockRuntime, provider: ActorID, id: ClaimID) -> Option { let st: State = rt.get_state(); let mut claims = st.load_claims(rt.store()).unwrap(); diff --git a/actors/verifreg/tests/verifreg_actor_test.rs b/actors/verifreg/tests/verifreg_actor_test.rs index ef81977d75..6a580a5326 100644 --- a/actors/verifreg/tests/verifreg_actor_test.rs +++ b/actors/verifreg/tests/verifreg_actor_test.rs @@ -495,6 +495,7 @@ mod allocs_claims { use cid::Cid; use fvm_ipld_encoding::ipld_block::IpldBlock; + use fvm_ipld_encoding::RawBytes; use fvm_shared::bigint::BigInt; use fvm_shared::error::ExitCode; use fvm_shared::piece::PaddedPieceSize; @@ -502,8 +503,8 @@ mod allocs_claims { use num_traits::Zero; use fil_actor_verifreg::{ - Actor, AllocationID, ClaimTerm, DataCap, ExtendClaimTermsParams, GetClaimsParams, - GetClaimsReturn, Method, State, + Actor, AllocationID, AllocationKey, ClaimKey, ClaimTerm, DataCap, ExtendClaimTermsParams, + GetClaimsParams, GetClaimsReturn, Method, State, }; use fil_actor_verifreg::{Claim, ExtendClaimTermsReturn}; use fil_actors_runtime::runtime::policy_constants::{ @@ -511,7 +512,8 @@ mod allocs_claims { MINIMUM_VERIFIED_ALLOCATION_TERM, }; use fil_actors_runtime::test_utils::{ - expect_abort, expect_abort_contains_message, ACCOUNT_ACTOR_CODE_ID, EVM_ACTOR_CODE_ID, + expect_abort, expect_abort_contains_message, MockRuntime, ACCOUNT_ACTOR_CODE_ID, + EVM_ACTOR_CODE_ID, }; use fil_actors_runtime::FailCode; use harness::*; @@ -520,6 +522,7 @@ mod allocs_claims { const CLIENT1: ActorID = 101; const CLIENT2: ActorID = 102; + const CLIENT3: ActorID = 103; const PROVIDER1: ActorID = 301; const PROVIDER2: ActorID = 302; const ALLOC_SIZE: u64 = MINIMUM_VERIFIED_ALLOCATION_SIZE as u64; @@ -1112,6 +1115,149 @@ mod allocs_claims { h.check_state(&rt); } + + #[test] + fn list_allocs_basics() { + let (h, rt) = new_harness(); + let start = &RawBytes::default(); + + // Empty. + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[], &allocs); + assert_eq!(None, next); + + // Singleton + let k1 = alloc(&rt, &h, CLIENT1, PROVIDER1, "1"); + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[k1], &allocs); + assert_eq!(None, next); + + // Same client + let k2 = alloc(&rt, &h, CLIENT1, PROVIDER2, "2"); + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[k1, k2], &allocs); + assert_eq!(None, next); + + // Different client + let k3 = alloc(&rt, &h, CLIENT2, PROVIDER1, "3"); + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[k1, k2, k3], &allocs); + assert_eq!(None, next); + } + + #[test] + fn list_allocs_pagination() { + let (h, rt) = new_harness(); + let start = &RawBytes::default(); + let keys = vec![ + alloc(&rt, &h, CLIENT1, PROVIDER1, "1"), + alloc(&rt, &h, CLIENT2, PROVIDER2, "2"), + alloc(&rt, &h, CLIENT2, PROVIDER1, "3"), + alloc(&rt, &h, CLIENT3, PROVIDER2, "4"), + alloc(&rt, &h, CLIENT3, PROVIDER1, "5"), + alloc(&rt, &h, CLIENT3, PROVIDER2, "6"), + ]; + + { + // Limit zero from the start returns a usable cursor + let (allocs, next) = h.list_allocs(&rt, start, 0); + assert_allocs(&[], &allocs); + assert!(next.is_some()); + + let (allocs, next2) = h.list_allocs(&rt, next.as_ref().unwrap(), 0); + assert_allocs(&[], &allocs); + assert_eq!(next, next2); + + let (allocs, next3) = h.list_allocs(&rt, next.as_ref().unwrap(), 100); + assert_allocs(&keys, &allocs); + assert!(next3.is_none()); + } + { + // Traverse N at a time. + for n in 1..=keys.len() { + let mut cursor = Some(start.clone()); + let mut collected = vec![]; + while cursor.is_some() { + let (allocs, next) = h.list_allocs(&rt, cursor.as_ref().unwrap(), n as u64); + collected.extend(allocs); + cursor = next + } + assert_allocs(&keys, &collected); + } + } + } + + #[test] + fn list_claims() { + // Listing claims and allocations uses the same code over different structures, + // so this test is a just a basic check that the right structure is used. + let (h, rt) = new_harness(); + let start = &RawBytes::default(); + let keys = vec![ + claim(&rt, &h, CLIENT1, PROVIDER1, "1"), + claim(&rt, &h, CLIENT1, PROVIDER2, "1"), + claim(&rt, &h, CLIENT2, PROVIDER2, "1"), + ]; + + // All at once. + let (claims, next) = h.list_claims(&rt, start, 100); + assert_claims(&keys, &claims); + assert_eq!(None, next); + + // One at a time. + let mut cursor = Some(start.clone()); + let mut collected = vec![]; + while cursor.is_some() { + let (allocs, next) = h.list_claims(&rt, cursor.as_ref().unwrap(), 1); + collected.extend(allocs); + cursor = next + } + assert_claims(&keys, &collected); + } + + fn alloc( + rt: &MockRuntime, + h: &Harness, + client: ActorID, + provider: ActorID, + data: &str, + ) -> AllocationKey { + let id = h.create_alloc(rt, &make_alloc(data, client, provider, ALLOC_SIZE)).unwrap(); + AllocationKey { client, id } + } + + fn claim( + rt: &MockRuntime, + h: &Harness, + client: ActorID, + provider: ActorID, + data: &str, + ) -> ClaimKey { + let id = h + .create_claim(rt, &make_claim(data, client, provider, ALLOC_SIZE, 0, 100, 0, 0)) + .unwrap(); + ClaimKey { provider, id } + } + + // Asserts that two collections of allocation keys have the same entries, regardless of order. + fn assert_allocs(expected: &[AllocationKey], actual: &[AllocationKey]) { + assert_eq!(expected.len(), actual.len()); + let mut expected = expected.to_vec(); + expected.sort(); + let mut actual = actual.to_vec(); + actual.sort(); + assert_eq!(expected, actual); + } + + // Asserts that two collections of claim keys have the same entries, regardless of order. + fn assert_claims(expected: &[ClaimKey], actual: &[ClaimKey]) { + assert_eq!(expected.len(), actual.len()); + let mut expected = expected.to_vec(); + expected.sort(); + let mut actual = actual.to_vec(); + actual.sort(); + assert_eq!(expected, actual); + } } mod datacap { diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index c0f8f352fa..2b5e20bea8 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -8,10 +8,10 @@ use fvm_ipld_blockstore::Blockstore; use fvm_ipld_hamt::Sha256; use fvm_ipld_hamt::{BytesKey, Error as HamtError, Hamt}; use fvm_shared::bigint::BigInt; +use fvm_shared::error::ExitCode; pub use fvm_shared::BLOCKS_PER_EPOCH as EXPECTED_LEADERS_PER_EPOCH; use serde::de::DeserializeOwned; use serde::Serialize; -use unsigned_varint::decode::Error as UVarintError; use builtin::HAMT_BIT_WIDTH; pub use dispatch::{dispatch, dispatch_default, WithCodec}; @@ -102,8 +102,9 @@ pub fn u64_key(k: u64) -> BytesKey { slice.into() } -pub fn parse_uint_key(s: &[u8]) -> Result { - let (v, _) = unsigned_varint::decode::u64(s)?; +pub fn parse_uint_key(s: &[u8]) -> Result { + let (v, _) = unsigned_varint::decode::u64(s) + .context_code(ExitCode::USR_SERIALIZATION, "failed to parse uint key")?; Ok(v) } diff --git a/runtime/src/util/mapmap.rs b/runtime/src/util/mapmap.rs index ef0048f863..29aa4c2982 100644 --- a/runtime/src/util/mapmap.rs +++ b/runtime/src/util/mapmap.rs @@ -110,7 +110,7 @@ where self.outer.for_each(f) } - // Runs a function over all values for one outer key. + // Runs a function over all entries for one outer key. pub fn for_each_in(&mut self, outside_k: K1, f: F) -> Result<(), Error> where F: FnMut(&BytesKey, &V) -> anyhow::Result<()>, @@ -122,6 +122,91 @@ where in_map.for_each(f) } + // Runs a function over all entries for all outer keys. + // Returns (outer, inner) keys with which to resume iteration, if more than + // limit entries were available. + pub fn for_each_each( + &mut self, + start_at: Option<&K1>, + start_at_inner: Option<&K2>, + limit: Option, + mut f: F, + ) -> Result, Error> + where + F: FnMut(&BytesKey, &BytesKey, &V) -> anyhow::Result<()>, + { + let limit = limit.unwrap_or(u64::MAX); + let mut count = 0; + let mut first_outer = true; + let outeritr = match start_at { + Some(k) => self.outer.iter_from(&k.key())?, + None => self.outer.iter(), + }; + for item in outeritr { + let (k1, inner_root) = item?; + let in_map = make_map_with_root_and_bitwidth::( + inner_root, + *self.outer.store(), + self.inner_bitwidth, + )?; + let inneritr = if first_outer { + // Use start-at-inner only for the first outer key. + match start_at_inner { + Some(k) => in_map.iter_from(&k.key())?, + None => in_map.iter(), + } + } else { + in_map.iter() + }; + first_outer = false; + for inner_item in inneritr { + let (k2, v) = inner_item?; + // Advance until ready to call f with one-past-the-end so that these + // keys can be returned as the cursor to resume with. + if count >= limit { + return Ok(Some((k1.clone(), k2.clone()))); + } + f(k1, k2, v)?; + count += 1; + } + } + // Exhausted iteration. + Ok(None) + } + + // pub fn for_each_each2( + // &mut self, + // start_at: Option<&K1>, + // start_at_inner: Option<&K2>, + // ) -> Result>, Error> { + // let outeritr = match start_at { + // Some(k) => self.outer.iter_from(&k.key())?, + // None => self.outer.iter(), + // }; + // Ok(outeritr.flat_map(|r| { + // match r { + // Ok((k1, inner_root)) => { + // let in_map = make_map_with_root_and_bitwidth::( + // inner_root, + // *self.outer.store(), + // self.inner_bitwidth, + // )?; + // let inneritr = match start_at_inner { + // Some(k) => in_map.iter_from(&k.key())?, + // None => in_map.iter(), + // }; + // inneritr.map(|r| { + // match r { + // Ok((k2, v)) => {Ok((k1, k2, v))}, + // Err(e) => Err(e) + // } + // }) + // } + // Err(e) => std::iter::once(Err(e)).map(|x|x) + // } + // })) + // } + // Puts a key value pair in the MapMap, overwriting any existing value. // Returns the previous value, if any. pub fn put(&mut self, outside_k: K1, inside_k: K2, value: V) -> Result, Error> {