From 41c638bcafd37cf2dfcad64e65eeae8849eedf80 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 16 Jan 2025 21:14:20 +0100 Subject: [PATCH 1/3] [AHM] Preimage migration Signed-off-by: Oliver Tale-Yazdi --- Cargo.lock | 1 + integration-tests/ahm/src/tests.rs | 10 +- pallets/ah-migrator/src/lib.rs | 28 +++- pallets/ah-migrator/src/preimage.rs | 86 ++++++++++ pallets/rc-migrator/Cargo.toml | 4 + pallets/rc-migrator/src/lib.rs | 51 +++++- pallets/rc-migrator/src/preimage.md | 2 +- pallets/rc-migrator/src/preimage.rs | 234 ++++++++++++++++++++++++++++ pallets/rc-migrator/src/proxy.rs | 74 ++++----- pallets/rc-migrator/src/types.rs | 15 ++ 10 files changed, 461 insertions(+), 44 deletions(-) create mode 100644 pallets/ah-migrator/src/preimage.rs create mode 100644 pallets/rc-migrator/src/preimage.rs diff --git a/Cargo.lock b/Cargo.lock index 4cc2fe9560..bbf1544a0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8865,6 +8865,7 @@ dependencies = [ "log", "pallet-balances", "pallet-multisig", + "pallet-preimage", "pallet-proxy", "pallet-staking", "parity-scale-codec", diff --git a/integration-tests/ahm/src/tests.rs b/integration-tests/ahm/src/tests.rs index 9bd7a621bd..9c6de9929c 100644 --- a/integration-tests/ahm/src/tests.rs +++ b/integration-tests/ahm/src/tests.rs @@ -36,6 +36,7 @@ use frame_support::{pallet_prelude::*, traits::*, weights::WeightMeter}; use pallet_rc_migrator::{MigrationStage, RcMigrationStage}; use polkadot_primitives::InboundDownwardMessage; use remote_externalities::RemoteExternalities; +use pallet_rc_migrator::types::PalletMigrationChecks; use polkadot_runtime::Runtime as Polkadot; @@ -47,8 +48,9 @@ async fn account_migration_works() { let para_id = ParaId::from(1000); // Simulate relay blocks and grab the DMP messages - let dmp_messages = rc.execute_with(|| { + let (dmp_messages, pre_check_payload) = rc.execute_with(|| { let mut dmps = Vec::new(); + let pre_check_payload = pallet_rc_migrator::preimage::PreimageChunkMigrator::::pre_check(); // Loop until no more DMPs are added and we had at least 1 loop { @@ -59,10 +61,10 @@ async fn account_migration_works() { dmps.extend(new_dmps); if RcMigrationStage::::get() == - pallet_rc_migrator::MigrationStage::ProxyMigrationDone + pallet_rc_migrator::MigrationStage::PreimageMigrationDone { log::info!("Migration done"); - break dmps; + break (dmps, pre_check_payload); } } }); @@ -88,6 +90,8 @@ async fn account_migration_works() { log::debug!("AH DMP messages left: {}", fp.storage.count); next_block_ah(); } + + pallet_rc_migrator::preimage::PreimageChunkMigrator::::post_check(pre_check_payload); // NOTE that the DMP queue is probably not empty because the snapshot that we use contains // some overweight ones. }); diff --git a/pallets/ah-migrator/src/lib.rs b/pallets/ah-migrator/src/lib.rs index 395eec341a..6d13bc4768 100644 --- a/pallets/ah-migrator/src/lib.rs +++ b/pallets/ah-migrator/src/lib.rs @@ -35,6 +35,7 @@ pub mod account; pub mod multisig; pub mod proxy; pub mod types; +pub mod preimage; pub use pallet::*; @@ -48,12 +49,13 @@ use frame_support::{ }; use frame_system::pallet_prelude::*; use pallet_balances::{AccountData, Reasons as LockReasons}; -use pallet_rc_migrator::{accounts::Account as RcAccount, multisig::*, proxy::*}; +use pallet_rc_migrator::{accounts::Account as RcAccount, multisig::*, proxy::*, preimage::*}; use sp_application_crypto::Ss58Codec; use sp_runtime::{ traits::{Convert, TryConvert}, AccountId32, }; +use sp_core::H256; use sp_std::prelude::*; /// The log target of this pallet. @@ -71,6 +73,7 @@ pub mod pallet { + pallet_balances::Config + pallet_multisig::Config + pallet_proxy::Config + + pallet_preimage::Config { /// The overarching event type. type RuntimeEvent: From> + IsType<::RuntimeEvent>; @@ -148,6 +151,18 @@ pub mod pallet { /// How many proxy announcements failed to integrate. count_bad: u32, }, + /// Received a batch of `RcPreimageChunk` that are going to be integrated. + PreimageChunkBatchReceived { + /// How many preimage chunks are in the batch. + count: u32, + }, + /// We processed a batch of `RcPreimageChunk` that we received. + PreimageChunkBatchProcessed { + /// How many preimage chunks were successfully integrated. + count_good: u32, + /// How many preimage chunks failed to integrate. + count_bad: u32, + }, } #[pallet::pallet] @@ -207,8 +222,19 @@ pub mod pallet { announcements: Vec>, ) -> DispatchResult { ensure_root(origin)?; + Self::do_receive_proxy_announcements(announcements).map_err(Into::into) } + + #[pallet::call_index(4)] + pub fn receive_preimage_chunks( + origin: OriginFor, + chunks: Vec, + ) -> DispatchResult { + ensure_root(origin)?; + + Self::do_receive_preimage_chunks(chunks).map_err(Into::into) + } } #[pallet::hooks] diff --git a/pallets/ah-migrator/src/preimage.rs b/pallets/ah-migrator/src/preimage.rs new file mode 100644 index 0000000000..6f5a57707e --- /dev/null +++ b/pallets/ah-migrator/src/preimage.rs @@ -0,0 +1,86 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{types::*, *}; +use pallet_rc_migrator::preimage::alias; + +impl Pallet { + pub fn do_receive_preimage_chunks(chunks: Vec) -> Result<(), Error> { + Self::deposit_event(Event::PreimageChunkBatchReceived { count: chunks.len() as u32 }); + let (mut count_good, mut count_bad) = (0, 0); + log::info!(target: LOG_TARGET, "Integrating {} preimage chunks", chunks.len()); + + for chunk in chunks { + match Self::do_receive_preimage_chunk(chunk) { + Ok(()) => count_good += 1, + Err(e) => { + count_bad += 1; + log::error!(target: LOG_TARGET, "Error while integrating preimage chunk: {:?}", e); + }, + } + } + Self::deposit_event(Event::PreimageChunkBatchProcessed { count_good, count_bad }); + + Ok(()) + } + + pub fn do_receive_preimage_chunk(chunk: RcPreimageChunk) -> Result<(), Error> { + log::debug!(target: LOG_TARGET, "Integrating preimage chunk {} offset {}/{}", chunk.preimage_hash, chunk.chunk_byte_offset + chunk.chunk_bytes.len() as u32, chunk.preimage_len); + let key = (chunk.preimage_hash, chunk.preimage_len); + + // First check that we did not miss a chunk + let preimage = match alias::PreimageFor::::get(&key) { + Some(mut preimage) => { + if preimage.len() != chunk.chunk_byte_offset as usize { + defensive!("Preimage chunk missing"); + return Err(Error::::TODO); + } + + match preimage.try_mutate(|p| { + p.extend(chunk.chunk_bytes.clone()); + }) { + Some(preimage) => { + alias::PreimageFor::::insert(&key, &preimage); + preimage + }, + None => { + defensive!("Preimage too big"); + return Err(Error::::TODO); + } + } + }, + None => { + if chunk.chunk_byte_offset != 0 { + defensive!("Preimage chunk missing"); + return Err(Error::::TODO); + } + + let preimage: BoundedVec> = chunk.chunk_bytes; + debug_assert!(pallet_rc_migrator::preimage::CHUNK_SIZE <= pallet_rc_migrator::preimage::alias::MAX_SIZE); + let bounded_preimage: BoundedVec> = preimage.into_inner().try_into().expect("Asserted"); + alias::PreimageFor::::insert(key, &bounded_preimage); + bounded_preimage + } + }; + + if preimage.len() == chunk.preimage_len as usize + chunk.chunk_byte_offset as usize { + log::debug!(target: LOG_TARGET, "Preimage complete: {}", chunk.preimage_hash); + } + + Ok(()) + } +} diff --git a/pallets/rc-migrator/Cargo.toml b/pallets/rc-migrator/Cargo.toml index a57641160a..58748d1297 100644 --- a/pallets/rc-migrator/Cargo.toml +++ b/pallets/rc-migrator/Cargo.toml @@ -23,6 +23,7 @@ pallet-balances = { workspace = true } pallet-staking = { workspace = true } pallet-proxy = { workspace = true } pallet-multisig = { workspace = true } +pallet-preimage = { workspace = true } polkadot-runtime-common = { workspace = true } runtime-parachains = { workspace = true } polkadot-parachain-primitives = { workspace = true } @@ -51,6 +52,7 @@ std = [ "sp-runtime/std", "sp-std/std", "xcm/std", + "pallet-preimage/std" ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", @@ -64,6 +66,7 @@ runtime-benchmarks = [ "polkadot-runtime-common/runtime-benchmarks", "runtime-parachains/runtime-benchmarks", "sp-runtime/runtime-benchmarks", + "pallet-preimage/runtime-benchmarks" ] try-runtime = [ "frame-support/try-runtime", @@ -75,4 +78,5 @@ try-runtime = [ "polkadot-runtime-common/try-runtime", "runtime-parachains/try-runtime", "sp-runtime/try-runtime", + "pallet-preimage/try-runtime" ] diff --git a/pallets/rc-migrator/src/lib.rs b/pallets/rc-migrator/src/lib.rs index 33e2c82429..7729277483 100644 --- a/pallets/rc-migrator/src/lib.rs +++ b/pallets/rc-migrator/src/lib.rs @@ -35,6 +35,7 @@ pub mod accounts; pub mod multisig; pub mod proxy; pub mod types; +pub mod preimage; mod weights; pub use pallet::*; @@ -49,6 +50,7 @@ use frame_support::{ }, weights::WeightMeter, }; +use sp_core::H256; use frame_system::{pallet_prelude::*, AccountInfo}; use pallet_balances::AccountData; use polkadot_parachain_primitives::primitives::Id as ParaId; @@ -65,6 +67,7 @@ use xcm::prelude::*; use multisig::MultisigMigrator; use proxy::*; use types::PalletMigration; +use preimage::PreimageChunkMigrator; /// The log target of this pallet. pub const LOG_TARGET: &str = "runtime::rc-migrator"; @@ -103,6 +106,13 @@ pub enum MigrationStage { last_key: Option, }, ProxyMigrationDone, + PreimageMigrationInit, + PreimageMigrationChunksOngoing { + // TODO type + last_key: Option<(Option<(H256, u32)>, u32)>, + }, + PreimageMigrationChunksDone, + PreimageMigrationDone, } type AccountInfoFor = @@ -125,6 +135,7 @@ pub mod pallet { + paras_registrar::Config + pallet_multisig::Config + pallet_proxy::Config + + pallet_preimage::Config { /// The overarching event type. type RuntimeEvent: From> + IsType<::RuntimeEvent>; @@ -220,8 +231,9 @@ pub mod pallet { // TODO: not complete let _ = Self::obtain_rc_accounts(); - Self::transition(MigrationStage::AccountsMigrationInit); - //Self::transition(MigrationStage::ProxyMigrationInit); + // Swap this out if you want to skip some migrations for faster debugging + //Self::transition(MigrationStage::AccountsMigrationInit); + Self::transition(MigrationStage::PreimageMigrationInit); }, MigrationStage::AccountsMigrationInit => { let first_acc = match Self::first_account(&mut weight_counter).defensive() { @@ -359,7 +371,38 @@ pub mod pallet { } }, MigrationStage::ProxyMigrationDone => { - todo!(); + Self::transition(MigrationStage::PreimageMigrationInit); + }, + MigrationStage::PreimageMigrationInit => { + Self::transition(MigrationStage::PreimageMigrationChunksOngoing { last_key: None }); + }, + MigrationStage::PreimageMigrationChunksOngoing { last_key } => { + let res = with_transaction_opaque_err::, Error, _>(|| { + TransactionOutcome::Commit(PreimageChunkMigrator::::migrate_many( + last_key, + &mut weight_counter, + )) + }) + .expect("Always returning Ok; qed"); + + match res { + Ok(None) => { + Self::transition(MigrationStage::PreimageMigrationDone); + }, + Ok(Some(last_key)) => { + Self::transition(MigrationStage::PreimageMigrationChunksOngoing { last_key: Some(last_key) }); + }, + e => { + log::error!(target: LOG_TARGET, "Error while migrating preimages: {:?}", e); + defensive!("Error while migrating preimages"); + }, + } + }, + MigrationStage::PreimageMigrationChunksDone => { + Self::transition(MigrationStage::PreimageMigrationDone); + }, + MigrationStage::PreimageMigrationDone => { + unimplemented!() }, }; @@ -405,7 +448,7 @@ pub mod pallet { batch.push(items.pop().unwrap()); // FAIL-CI no unwrap } - log::info!(target: LOG_TARGET, "Sending batch of {} proxies", batch.len()); + log::info!(target: LOG_TARGET, "Sending XCM batch of {} items", batch.len()); let call = types::AssetHubPalletConfig::::AhmController(create_call(batch)); let message = Xcm(vec![ diff --git a/pallets/rc-migrator/src/preimage.md b/pallets/rc-migrator/src/preimage.md index 2e3ef4df41..c8198d1d62 100644 --- a/pallets/rc-migrator/src/preimage.md +++ b/pallets/rc-migrator/src/preimage.md @@ -23,7 +23,7 @@ Deprecated. Will not be migrated but funds will be unreserved. For anyone who has registered a preimage: - If the preimage was in the new RequestStatusFor: Some unlocked funds 😎. We cannot calculate a list of affected accounts in advance since users can still influence this. -- If the preimage was in the old StatusFor: will be removed and funds unlocked. Exhaustive list of all 166 Polkadot accounts that are affected by this and will have **UP TO** these funds unlocked (not a legally binding statement): +- If the preimage was in the old StatusFor: will be removed and funds unlocked. [Exhaustive list](https://github.com/ggwpez/substrate-scripts/blob/master/ahm-preimage-statusfor-accounts.py) of all 166 Polkadot accounts that are affected by this and will have **UP TO** these funds unlocked (not a legally binding statement): - `16LKv69ct6xDzSiUjuz154vCg62dkyysektHFCeJe85xb6X`: 1256.897 DOT - `15ynbcMgPf7HbQErRz66RDLMuBVdcWVuURhR4SLPiqa6B8jx`: 633.121 DOT diff --git a/pallets/rc-migrator/src/preimage.rs b/pallets/rc-migrator/src/preimage.rs new file mode 100644 index 0000000000..1f43555aa4 --- /dev/null +++ b/pallets/rc-migrator/src/preimage.rs @@ -0,0 +1,234 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc = include_str!("multisig.md")] + +use crate::{*, types::*}; +use sp_runtime::traits::BlakeTwo256; +use sp_runtime::traits::Hash; + +pub mod alias { + use super::*; + + use sp_core::ConstU32; + use frame_support::Identity; + use frame_support::traits::Currency; + + pub const MAX_SIZE: u32 = 4 * 1024 * 1024; + + /// A type to note whether a preimage is owned by a user or the system. + // Copied from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L67-L77 + #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] + pub enum OldRequestStatus { + /// The associated preimage has not yet been requested by the system. The given deposit (if + /// some) is being held until either it becomes requested or the user retracts the preimage. + Unrequested { deposit: (AccountId, Balance), len: u32 }, + /// There are a non-zero number of outstanding requests for this hash by this chain. If there + /// is a preimage registered, then `len` is `Some` and it may be removed iff this counter + /// becomes zero. + Requested { deposit: Option<(AccountId, Balance)>, count: u32, len: Option }, + } + + /// A type to note whether a preimage is owned by a user or the system. + // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L79-L89 + #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] + pub enum RequestStatus { + /// The associated preimage has not yet been requested by the system. The given deposit (if + /// some) is being held until either it becomes requested or the user retracts the preimage. + Unrequested { ticket: (AccountId, Ticket), len: u32 }, + /// There are a non-zero number of outstanding requests for this hash by this chain. If there + /// is a preimage registered, then `len` is `Some` and it may be removed iff this counter + /// becomes zero. + Requested { maybe_ticket: Option<(AccountId, Ticket)>, count: u32, maybe_len: Option }, + } + + // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L91-L93 + type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + type TicketOf = ::Consideration; + + // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L173-L185 + #[deprecated = "RequestStatusFor"] + #[frame_support::storage_alias(pallet_name)] + pub type StatusFor = + StorageMap, Identity, H256, OldRequestStatus<::AccountId, BalanceOf>>; + + #[frame_support::storage_alias(pallet_name)] + pub type RequestStatusFor = + StorageMap, Identity, H256, RequestStatus<::AccountId, TicketOf>>; + + #[frame_support::storage_alias(pallet_name)] + pub type PreimageFor = + StorageMap, Identity, (H256, u32), BoundedVec>>; +} + +pub const CHUNK_SIZE: u32 = 49_900; // about 50KiB + +/// A chunk of a preimage that was migrated out of the Relay and can be integrated into AH. +#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] +pub struct RcPreimageChunk { + /// The hash of the original preimage. + pub preimage_hash: H256, + /// The length of the original preimage. + pub preimage_len: u32, + /// Where this chunk starts in the original preimage. + pub chunk_byte_offset: u32, + /// A chunk of the original preimage. + pub chunk_bytes: BoundedVec>, +} + +/// An entry of the `RequestStatusFor` storage map. +#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] +pub struct RcPreimageRequestStatus { + /// The hash of the original preimage. + pub hash: H256, + /// The request status of the original preimage. + pub request_status: alias::RequestStatus, +} + +/// An entry of the `StatusFor` storage map. Should only be used to unreserve funds on AH. +#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] +pub struct RcPreimageLegacyStatus { + /// The hash of the original preimage. + pub hash: H256, + /// The request status of the original preimage. + pub request_status: alias::OldRequestStatus, +} + +pub struct PreimageChunkMigrator { + _phantom: PhantomData, +} + +impl PalletMigration for PreimageChunkMigrator { + type Key = (Option<(H256, u32)>, u32); + type Error = Error; + + fn migrate_many( + mut last_key: Option, + weight_counter: &mut WeightMeter, + ) -> Result, Self::Error> { + let (key_iter, mut last_offset) = match last_key { + None => (alias::PreimageFor::::iter_keys(), 0), + Some((None, offset)) => (alias::PreimageFor::::iter_keys(), offset), + Some((Some((hash, len)), offset)) => ( + alias::PreimageFor::::iter_keys_from( + alias::PreimageFor::::hashed_key_for(&(hash, len)) + ), + offset + ), + }; + + let mut batch = Vec::new(); + let mut current_key = None; + + for kv in key_iter { + // If we're starting a new preimage, reset offset + if current_key.as_ref() != Some(&kv) { + current_key = Some(kv.clone()); + // Reset offset unless we're resuming this specific hash from a previous attempt + let should_reset = last_key + .as_ref() + .and_then(|k| k.0.as_ref()) + .map_or(true, |h| h != &kv); + + if should_reset { + last_offset = 0; + } + } + + // Get the full preimage data once + let full_data = alias::PreimageFor::::get(&kv).unwrap_or_default(); + + // Process chunks while there's still data to process + while last_offset < kv.1 { + // Calculate how many bytes remain to be processed + let remaining_bytes = kv.1.saturating_sub(last_offset); + let chunk_size = remaining_bytes.min(CHUNK_SIZE); + + // Extract the chunk + let chunk_bytes: Vec = full_data + .iter() + .skip(last_offset as usize) + .take(chunk_size as usize) + .cloned() + .collect(); + + let bounded_chunk_bytes = BoundedVec::try_from(chunk_bytes) + .expect("Chunk size is bounded by CHUNK_SIZE; qed"); + debug_assert!(bounded_chunk_bytes.len() == chunk_size as usize); + + batch.push(RcPreimageChunk { + preimage_hash: kv.0, + preimage_len: kv.1, + chunk_byte_offset: last_offset, + chunk_bytes: bounded_chunk_bytes, + }); + + log::debug!( + target: LOG_TARGET, + "Processed preimage chunk {:?} at offset {}", + kv, + last_offset + ); + + last_offset += chunk_size; + + // Return after processing 10 chunks, saving our progress + if batch.len() >= 10 { + Pallet::::send_chunked_xcm(batch, |batch| { + types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } + })?; + return Ok(Some((Some(kv), last_offset))); + } + } + + // After finishing a preimage, update last_key and reset offset + last_key = Some((Some(kv), 0)); + last_offset = 0; + } + + // Send any remaining batch before finishing + if !batch.is_empty() { + Pallet::::send_chunked_xcm(batch, |batch| { + types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } + })?; + } + + Ok(None) // No more preimages to process + } +} + +impl PalletMigrationChecks for PreimageChunkMigrator { + type Payload = Vec<(H256, u32)>; + + fn pre_check() -> Self::Payload { + alias::PreimageFor::::iter_keys().collect() + } + + fn post_check(keys: Self::Payload) { + // Check that all keys are inserted + for (hash, len) in keys { + assert!(alias::PreimageFor::::contains_key(&(hash, len))); + } + // Integrity check that all preimages have the correct hash and length + for (hash, len) in alias::PreimageFor::::iter_keys() { + let preimage = alias::PreimageFor::::get(&(hash, len)).expect("Storage corrupted"); + + assert_eq!(preimage.len(), len as usize); + assert_eq!(BlakeTwo256::hash(preimage.as_slice()), hash); + } + } +} diff --git a/pallets/rc-migrator/src/proxy.rs b/pallets/rc-migrator/src/proxy.rs index 0c02d49da8..8eefad9ef4 100644 --- a/pallets/rc-migrator/src/proxy.rs +++ b/pallets/rc-migrator/src/proxy.rs @@ -70,38 +70,33 @@ impl PalletMigration for ProxyProxiesMigrator { weight_counter: &mut WeightMeter, ) -> Result>, Error> { let mut batch = Vec::new(); - + + // Get iterator starting after last processed key let mut key_iter = if let Some(last_key) = last_key.clone() { - pallet_proxy::Proxies::::iter_keys_from(pallet_proxy::Proxies::::hashed_key_for( - &last_key, - )) + pallet_proxy::Proxies::::iter_from( + pallet_proxy::Proxies::::hashed_key_for(&last_key) + ) } else { - pallet_proxy::Proxies::::iter_keys() + pallet_proxy::Proxies::::iter() }; - loop { - let Some(acc) = key_iter.next() else { - last_key = None; - log::info!(target: LOG_TARGET, "No more proxies to migrate, last key: {:?}", &last_key); - break; - }; - log::debug!("Migrating proxies of acc {:?}", acc); - - let (proxies, deposit) = pallet_proxy::Proxies::::get(&acc); - + // Process accounts until we run out of weight or accounts + while let Some((acc, (proxies, deposit))) = key_iter.next() { if proxies.is_empty() { defensive!("The proxy pallet disallows empty proxy lists"); continue; - }; + } - match Self::migrate_single(acc.clone(), (proxies.into_inner(), deposit), weight_counter) - { - Ok(proxy) => batch.push(proxy), - Err(Error::OutOfWeight) if batch.len() > 0 => { - log::info!(target: LOG_TARGET, "Out of weight, continuing with next batch"); + match Self::migrate_single(acc.clone(), (proxies.into_inner(), deposit), weight_counter) { + Ok(proxy) => { + batch.push(proxy); + last_key = Some(acc); // Update last processed key + }, + Err(Error::OutOfWeight) if !batch.is_empty() => { + // We have items to process but ran out of weight break; }, - Err(Error::OutOfWeight) if batch.len() == 0 => { + Err(Error::OutOfWeight) => { defensive!("Not enough weight to migrate a single account"); return Err(Error::OutOfWeight); }, @@ -111,19 +106,21 @@ impl PalletMigration for ProxyProxiesMigrator { return Err(e); }, } - - last_key = Some(acc); // Mark as successfully migrated } - // TODO send xcm + // Send batch if we have any items if !batch.is_empty() { Pallet::::send_chunked_xcm(batch, |batch| { types::AhMigratorCall::::ReceiveProxyProxies { proxies: batch } })?; } - log::info!(target: LOG_TARGET, "Last key: {:?}", &last_key); - Ok(last_key) + // Return last processed key if there are more items, None if we're done + if key_iter.next().is_some() { + Ok(last_key) + } else { + Ok(None) + } } } @@ -163,33 +160,40 @@ impl PalletMigration for ProxyAnnouncementMigrator { last_key: Option, weight_counter: &mut WeightMeter, ) -> Result, Self::Error> { - if weight_counter.try_consume(Weight::from_all(1_000)).is_err() { - return Err(Error::::OutOfWeight); - } - let mut batch = Vec::new(); + let mut last_processed = None; + + // Get iterator starting after last processed key let mut iter = if let Some(last_key) = last_key { pallet_proxy::Announcements::::iter_from( - pallet_proxy::Announcements::::hashed_key_for(&last_key), + pallet_proxy::Announcements::::hashed_key_for(&last_key) ) } else { pallet_proxy::Announcements::::iter() }; - while let Some((acc, (_announcements, deposit))) = iter.next() { + // Process announcements until we run out of weight + while let Some((acc, (announcements, deposit))) = iter.next() { if weight_counter.try_consume(Weight::from_all(1_000)).is_err() { break; } - batch.push(RcProxyAnnouncement { depositor: acc, deposit }); + batch.push(RcProxyAnnouncement { depositor: acc.clone(), deposit }); + last_processed = Some(acc); } + // Send batch if we have any items if !batch.is_empty() { Pallet::::send_chunked_xcm(batch, |batch| { types::AhMigratorCall::::ReceiveProxyAnnouncements { announcements: batch } })?; } - Ok(None) + // Return last processed key if there are more items, None if we're done + if iter.next().is_some() { + Ok(last_processed) + } else { + Ok(None) + } } } diff --git a/pallets/rc-migrator/src/types.rs b/pallets/rc-migrator/src/types.rs index e8e88f4b2b..e385e2f6fc 100644 --- a/pallets/rc-migrator/src/types.rs +++ b/pallets/rc-migrator/src/types.rs @@ -38,6 +38,8 @@ pub enum AhMigratorCall { ReceiveProxyProxies { proxies: Vec> }, #[codec(index = 3)] ReceiveProxyAnnouncements { announcements: Vec> }, + #[codec(index = 4)] + ReceivePreimageChunks { chunks: Vec }, } /// Copy of `ParaInfo` type from `paras_registrar` pallet. @@ -79,3 +81,16 @@ pub trait PalletMigration { weight_counter: &mut WeightMeter, ) -> Result, Self::Error>; } + +/// Trait to run some checks before and after a pallet migration. +/// +/// This needs to be called by the test harness. +pub trait PalletMigrationChecks { + type Payload; + + /// Run some checks before the migration and store intermediate payload. + fn pre_check() -> Self::Payload; + + /// Run some checks after the migration and use the intermediate payload. + fn post_check(payload: Self::Payload); +} From f935da0f388fc1042683add5fe600f55a0cce7a7 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Thu, 16 Jan 2025 21:14:45 +0100 Subject: [PATCH 2/3] fmt Signed-off-by: Oliver Tale-Yazdi --- integration-tests/ahm/src/tests.rs | 10 +- pallets/ah-migrator/src/lib.rs | 6 +- pallets/ah-migrator/src/preimage.rs | 19 ++- pallets/rc-migrator/src/lib.rs | 15 +- pallets/rc-migrator/src/preimage.rs | 244 +++++++++++++++------------- pallets/rc-migrator/src/proxy.rs | 15 +- 6 files changed, 168 insertions(+), 141 deletions(-) diff --git a/integration-tests/ahm/src/tests.rs b/integration-tests/ahm/src/tests.rs index 9c6de9929c..51188b65cd 100644 --- a/integration-tests/ahm/src/tests.rs +++ b/integration-tests/ahm/src/tests.rs @@ -33,10 +33,9 @@ use cumulus_primitives_core::{AggregateMessageOrigin, ParaId}; use frame_support::{pallet_prelude::*, traits::*, weights::WeightMeter}; -use pallet_rc_migrator::{MigrationStage, RcMigrationStage}; +use pallet_rc_migrator::{types::PalletMigrationChecks, MigrationStage, RcMigrationStage}; use polkadot_primitives::InboundDownwardMessage; use remote_externalities::RemoteExternalities; -use pallet_rc_migrator::types::PalletMigrationChecks; use polkadot_runtime::Runtime as Polkadot; @@ -50,7 +49,8 @@ async fn account_migration_works() { // Simulate relay blocks and grab the DMP messages let (dmp_messages, pre_check_payload) = rc.execute_with(|| { let mut dmps = Vec::new(); - let pre_check_payload = pallet_rc_migrator::preimage::PreimageChunkMigrator::::pre_check(); + let pre_check_payload = + pallet_rc_migrator::preimage::PreimageChunkMigrator::::pre_check(); // Loop until no more DMPs are added and we had at least 1 loop { @@ -91,7 +91,9 @@ async fn account_migration_works() { next_block_ah(); } - pallet_rc_migrator::preimage::PreimageChunkMigrator::::post_check(pre_check_payload); + pallet_rc_migrator::preimage::PreimageChunkMigrator::::post_check( + pre_check_payload, + ); // NOTE that the DMP queue is probably not empty because the snapshot that we use contains // some overweight ones. }); diff --git a/pallets/ah-migrator/src/lib.rs b/pallets/ah-migrator/src/lib.rs index 6d13bc4768..0b4554e30d 100644 --- a/pallets/ah-migrator/src/lib.rs +++ b/pallets/ah-migrator/src/lib.rs @@ -33,9 +33,9 @@ pub mod account; pub mod multisig; +pub mod preimage; pub mod proxy; pub mod types; -pub mod preimage; pub use pallet::*; @@ -49,13 +49,13 @@ use frame_support::{ }; use frame_system::pallet_prelude::*; use pallet_balances::{AccountData, Reasons as LockReasons}; -use pallet_rc_migrator::{accounts::Account as RcAccount, multisig::*, proxy::*, preimage::*}; +use pallet_rc_migrator::{accounts::Account as RcAccount, multisig::*, preimage::*, proxy::*}; use sp_application_crypto::Ss58Codec; +use sp_core::H256; use sp_runtime::{ traits::{Convert, TryConvert}, AccountId32, }; -use sp_core::H256; use sp_std::prelude::*; /// The log target of this pallet. diff --git a/pallets/ah-migrator/src/preimage.rs b/pallets/ah-migrator/src/preimage.rs index 6f5a57707e..163c7d0935 100644 --- a/pallets/ah-migrator/src/preimage.rs +++ b/pallets/ah-migrator/src/preimage.rs @@ -60,7 +60,7 @@ impl Pallet { None => { defensive!("Preimage too big"); return Err(Error::::TODO); - } + }, } }, None => { @@ -69,12 +69,21 @@ impl Pallet { return Err(Error::::TODO); } - let preimage: BoundedVec> = chunk.chunk_bytes; - debug_assert!(pallet_rc_migrator::preimage::CHUNK_SIZE <= pallet_rc_migrator::preimage::alias::MAX_SIZE); - let bounded_preimage: BoundedVec> = preimage.into_inner().try_into().expect("Asserted"); + let preimage: BoundedVec< + u8, + ConstU32<{ pallet_rc_migrator::preimage::CHUNK_SIZE }>, + > = chunk.chunk_bytes; + debug_assert!( + pallet_rc_migrator::preimage::CHUNK_SIZE <= + pallet_rc_migrator::preimage::alias::MAX_SIZE + ); + let bounded_preimage: BoundedVec< + u8, + ConstU32<{ pallet_rc_migrator::preimage::alias::MAX_SIZE }>, + > = preimage.into_inner().try_into().expect("Asserted"); alias::PreimageFor::::insert(key, &bounded_preimage); bounded_preimage - } + }, }; if preimage.len() == chunk.preimage_len as usize + chunk.chunk_byte_offset as usize { diff --git a/pallets/rc-migrator/src/lib.rs b/pallets/rc-migrator/src/lib.rs index 7729277483..69b965e21c 100644 --- a/pallets/rc-migrator/src/lib.rs +++ b/pallets/rc-migrator/src/lib.rs @@ -33,9 +33,9 @@ pub mod accounts; pub mod multisig; +pub mod preimage; pub mod proxy; pub mod types; -pub mod preimage; mod weights; pub use pallet::*; @@ -50,13 +50,12 @@ use frame_support::{ }, weights::WeightMeter, }; -use sp_core::H256; use frame_system::{pallet_prelude::*, AccountInfo}; use pallet_balances::AccountData; use polkadot_parachain_primitives::primitives::Id as ParaId; use polkadot_runtime_common::paras_registrar; use runtime_parachains::hrmp; -use sp_core::crypto::Ss58Codec; +use sp_core::{crypto::Ss58Codec, H256}; use sp_runtime::{traits::TryConvert, AccountId32}; use sp_std::prelude::*; use storage::TransactionOutcome; @@ -65,9 +64,9 @@ use weights::WeightInfo; use xcm::prelude::*; use multisig::MultisigMigrator; +use preimage::PreimageChunkMigrator; use proxy::*; use types::PalletMigration; -use preimage::PreimageChunkMigrator; /// The log target of this pallet. pub const LOG_TARGET: &str = "runtime::rc-migrator"; @@ -374,7 +373,9 @@ pub mod pallet { Self::transition(MigrationStage::PreimageMigrationInit); }, MigrationStage::PreimageMigrationInit => { - Self::transition(MigrationStage::PreimageMigrationChunksOngoing { last_key: None }); + Self::transition(MigrationStage::PreimageMigrationChunksOngoing { + last_key: None, + }); }, MigrationStage::PreimageMigrationChunksOngoing { last_key } => { let res = with_transaction_opaque_err::, Error, _>(|| { @@ -390,7 +391,9 @@ pub mod pallet { Self::transition(MigrationStage::PreimageMigrationDone); }, Ok(Some(last_key)) => { - Self::transition(MigrationStage::PreimageMigrationChunksOngoing { last_key: Some(last_key) }); + Self::transition(MigrationStage::PreimageMigrationChunksOngoing { + last_key: Some(last_key), + }); }, e => { log::error!(target: LOG_TARGET, "Error while migrating preimages: {:?}", e); diff --git a/pallets/rc-migrator/src/preimage.rs b/pallets/rc-migrator/src/preimage.rs index 1f43555aa4..3f232c70d1 100644 --- a/pallets/rc-migrator/src/preimage.rs +++ b/pallets/rc-migrator/src/preimage.rs @@ -17,16 +17,14 @@ #![doc = include_str!("multisig.md")] -use crate::{*, types::*}; -use sp_runtime::traits::BlakeTwo256; -use sp_runtime::traits::Hash; +use crate::{types::*, *}; +use sp_runtime::traits::{BlakeTwo256, Hash}; pub mod alias { use super::*; + use frame_support::{traits::Currency, Identity}; use sp_core::ConstU32; - use frame_support::Identity; - use frame_support::traits::Currency; pub const MAX_SIZE: u32 = 4 * 1024 * 1024; @@ -35,11 +33,12 @@ pub mod alias { #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] pub enum OldRequestStatus { /// The associated preimage has not yet been requested by the system. The given deposit (if - /// some) is being held until either it becomes requested or the user retracts the preimage. + /// some) is being held until either it becomes requested or the user retracts the + /// preimage. Unrequested { deposit: (AccountId, Balance), len: u32 }, - /// There are a non-zero number of outstanding requests for this hash by this chain. If there - /// is a preimage registered, then `len` is `Some` and it may be removed iff this counter - /// becomes zero. + /// There are a non-zero number of outstanding requests for this hash by this chain. If + /// there is a preimage registered, then `len` is `Some` and it may be removed iff this + /// counter becomes zero. Requested { deposit: Option<(AccountId, Balance)>, count: u32, len: Option }, } @@ -48,31 +47,46 @@ pub mod alias { #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] pub enum RequestStatus { /// The associated preimage has not yet been requested by the system. The given deposit (if - /// some) is being held until either it becomes requested or the user retracts the preimage. + /// some) is being held until either it becomes requested or the user retracts the + /// preimage. Unrequested { ticket: (AccountId, Ticket), len: u32 }, - /// There are a non-zero number of outstanding requests for this hash by this chain. If there - /// is a preimage registered, then `len` is `Some` and it may be removed iff this counter - /// becomes zero. + /// There are a non-zero number of outstanding requests for this hash by this chain. If + /// there is a preimage registered, then `len` is `Some` and it may be removed iff this + /// counter becomes zero. Requested { maybe_ticket: Option<(AccountId, Ticket)>, count: u32, maybe_len: Option }, } // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L91-L93 - type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + type BalanceOf = <::Currency as Currency< + ::AccountId, + >>::Balance; type TicketOf = ::Consideration; // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L173-L185 #[deprecated = "RequestStatusFor"] #[frame_support::storage_alias(pallet_name)] - pub type StatusFor = - StorageMap, Identity, H256, OldRequestStatus<::AccountId, BalanceOf>>; + pub type StatusFor = StorageMap< + pallet_preimage::Pallet, + Identity, + H256, + OldRequestStatus<::AccountId, BalanceOf>, + >; #[frame_support::storage_alias(pallet_name)] - pub type RequestStatusFor = - StorageMap, Identity, H256, RequestStatus<::AccountId, TicketOf>>; + pub type RequestStatusFor = StorageMap< + pallet_preimage::Pallet, + Identity, + H256, + RequestStatus<::AccountId, TicketOf>, + >; #[frame_support::storage_alias(pallet_name)] - pub type PreimageFor = - StorageMap, Identity, (H256, u32), BoundedVec>>; + pub type PreimageFor = StorageMap< + pallet_preimage::Pallet, + Identity, + (H256, u32), + BoundedVec>, + >; } pub const CHUNK_SIZE: u32 = 49_900; // about 50KiB @@ -113,102 +127,100 @@ pub struct PreimageChunkMigrator { } impl PalletMigration for PreimageChunkMigrator { - type Key = (Option<(H256, u32)>, u32); - type Error = Error; - - fn migrate_many( - mut last_key: Option, - weight_counter: &mut WeightMeter, - ) -> Result, Self::Error> { - let (key_iter, mut last_offset) = match last_key { - None => (alias::PreimageFor::::iter_keys(), 0), - Some((None, offset)) => (alias::PreimageFor::::iter_keys(), offset), - Some((Some((hash, len)), offset)) => ( - alias::PreimageFor::::iter_keys_from( - alias::PreimageFor::::hashed_key_for(&(hash, len)) - ), - offset - ), - }; - - let mut batch = Vec::new(); - let mut current_key = None; - - for kv in key_iter { - // If we're starting a new preimage, reset offset - if current_key.as_ref() != Some(&kv) { - current_key = Some(kv.clone()); - // Reset offset unless we're resuming this specific hash from a previous attempt - let should_reset = last_key - .as_ref() - .and_then(|k| k.0.as_ref()) - .map_or(true, |h| h != &kv); - - if should_reset { - last_offset = 0; - } - } - - // Get the full preimage data once - let full_data = alias::PreimageFor::::get(&kv).unwrap_or_default(); - - // Process chunks while there's still data to process - while last_offset < kv.1 { - // Calculate how many bytes remain to be processed - let remaining_bytes = kv.1.saturating_sub(last_offset); - let chunk_size = remaining_bytes.min(CHUNK_SIZE); - - // Extract the chunk - let chunk_bytes: Vec = full_data - .iter() - .skip(last_offset as usize) - .take(chunk_size as usize) - .cloned() - .collect(); - - let bounded_chunk_bytes = BoundedVec::try_from(chunk_bytes) - .expect("Chunk size is bounded by CHUNK_SIZE; qed"); - debug_assert!(bounded_chunk_bytes.len() == chunk_size as usize); - - batch.push(RcPreimageChunk { - preimage_hash: kv.0, - preimage_len: kv.1, - chunk_byte_offset: last_offset, - chunk_bytes: bounded_chunk_bytes, - }); - - log::debug!( - target: LOG_TARGET, - "Processed preimage chunk {:?} at offset {}", - kv, - last_offset - ); - - last_offset += chunk_size; - - // Return after processing 10 chunks, saving our progress - if batch.len() >= 10 { - Pallet::::send_chunked_xcm(batch, |batch| { - types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } - })?; - return Ok(Some((Some(kv), last_offset))); - } - } - - // After finishing a preimage, update last_key and reset offset - last_key = Some((Some(kv), 0)); - last_offset = 0; - } - - // Send any remaining batch before finishing - if !batch.is_empty() { - Pallet::::send_chunked_xcm(batch, |batch| { - types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } - })?; - } - - Ok(None) // No more preimages to process - } + type Key = (Option<(H256, u32)>, u32); + type Error = Error; + + fn migrate_many( + mut last_key: Option, + weight_counter: &mut WeightMeter, + ) -> Result, Self::Error> { + let (key_iter, mut last_offset) = match last_key { + None => (alias::PreimageFor::::iter_keys(), 0), + Some((None, offset)) => (alias::PreimageFor::::iter_keys(), offset), + Some((Some((hash, len)), offset)) => ( + alias::PreimageFor::::iter_keys_from(alias::PreimageFor::::hashed_key_for( + &(hash, len), + )), + offset, + ), + }; + + let mut batch = Vec::new(); + let mut current_key = None; + + for kv in key_iter { + // If we're starting a new preimage, reset offset + if current_key.as_ref() != Some(&kv) { + current_key = Some(kv.clone()); + // Reset offset unless we're resuming this specific hash from a previous attempt + let should_reset = + last_key.as_ref().and_then(|k| k.0.as_ref()).map_or(true, |h| h != &kv); + + if should_reset { + last_offset = 0; + } + } + + // Get the full preimage data once + let full_data = alias::PreimageFor::::get(&kv).unwrap_or_default(); + + // Process chunks while there's still data to process + while last_offset < kv.1 { + // Calculate how many bytes remain to be processed + let remaining_bytes = kv.1.saturating_sub(last_offset); + let chunk_size = remaining_bytes.min(CHUNK_SIZE); + + // Extract the chunk + let chunk_bytes: Vec = full_data + .iter() + .skip(last_offset as usize) + .take(chunk_size as usize) + .cloned() + .collect(); + + let bounded_chunk_bytes = BoundedVec::try_from(chunk_bytes) + .expect("Chunk size is bounded by CHUNK_SIZE; qed"); + debug_assert!(bounded_chunk_bytes.len() == chunk_size as usize); + + batch.push(RcPreimageChunk { + preimage_hash: kv.0, + preimage_len: kv.1, + chunk_byte_offset: last_offset, + chunk_bytes: bounded_chunk_bytes, + }); + + log::debug!( + target: LOG_TARGET, + "Processed preimage chunk {:?} at offset {}", + kv, + last_offset + ); + + last_offset += chunk_size; + + // Return after processing 10 chunks, saving our progress + if batch.len() >= 10 { + Pallet::::send_chunked_xcm(batch, |batch| { + types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } + })?; + return Ok(Some((Some(kv), last_offset))); + } + } + + // After finishing a preimage, update last_key and reset offset + last_key = Some((Some(kv), 0)); + last_offset = 0; + } + + // Send any remaining batch before finishing + if !batch.is_empty() { + Pallet::::send_chunked_xcm(batch, |batch| { + types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } + })?; + } + + Ok(None) // No more preimages to process + } } impl PalletMigrationChecks for PreimageChunkMigrator { diff --git a/pallets/rc-migrator/src/proxy.rs b/pallets/rc-migrator/src/proxy.rs index 8eefad9ef4..4b1256a474 100644 --- a/pallets/rc-migrator/src/proxy.rs +++ b/pallets/rc-migrator/src/proxy.rs @@ -70,12 +70,12 @@ impl PalletMigration for ProxyProxiesMigrator { weight_counter: &mut WeightMeter, ) -> Result>, Error> { let mut batch = Vec::new(); - + // Get iterator starting after last processed key let mut key_iter = if let Some(last_key) = last_key.clone() { - pallet_proxy::Proxies::::iter_from( - pallet_proxy::Proxies::::hashed_key_for(&last_key) - ) + pallet_proxy::Proxies::::iter_from(pallet_proxy::Proxies::::hashed_key_for( + &last_key, + )) } else { pallet_proxy::Proxies::::iter() }; @@ -87,7 +87,8 @@ impl PalletMigration for ProxyProxiesMigrator { continue; } - match Self::migrate_single(acc.clone(), (proxies.into_inner(), deposit), weight_counter) { + match Self::migrate_single(acc.clone(), (proxies.into_inner(), deposit), weight_counter) + { Ok(proxy) => { batch.push(proxy); last_key = Some(acc); // Update last processed key @@ -162,11 +163,11 @@ impl PalletMigration for ProxyAnnouncementMigrator { ) -> Result, Self::Error> { let mut batch = Vec::new(); let mut last_processed = None; - + // Get iterator starting after last processed key let mut iter = if let Some(last_key) = last_key { pallet_proxy::Announcements::::iter_from( - pallet_proxy::Announcements::::hashed_key_for(&last_key) + pallet_proxy::Announcements::::hashed_key_for(&last_key), ) } else { pallet_proxy::Announcements::::iter() From b526c3243d08d48a4e6672ad360bf2b246453a1d Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Fri, 17 Jan 2025 23:15:19 +0100 Subject: [PATCH 3/3] continue with preimage migration Signed-off-by: Oliver Tale-Yazdi --- integration-tests/ahm/src/tests.rs | 2 +- pallets/ah-migrator/src/lib.rs | 22 ++ pallets/ah-migrator/src/preimage.rs | 92 ++++++- pallets/rc-migrator/src/lib.rs | 47 +++- pallets/rc-migrator/src/preimage.rs | 246 ------------------ pallets/rc-migrator/src/preimage/chunks.rs | 162 ++++++++++++ pallets/rc-migrator/src/preimage/mod.rs | 105 ++++++++ .../src/{ => preimage}/preimage.md | 0 .../src/preimage/request_status.rs | 95 +++++++ pallets/rc-migrator/src/types.rs | 2 + 10 files changed, 513 insertions(+), 260 deletions(-) delete mode 100644 pallets/rc-migrator/src/preimage.rs create mode 100644 pallets/rc-migrator/src/preimage/chunks.rs create mode 100644 pallets/rc-migrator/src/preimage/mod.rs rename pallets/rc-migrator/src/{ => preimage}/preimage.md (100%) create mode 100644 pallets/rc-migrator/src/preimage/request_status.rs diff --git a/integration-tests/ahm/src/tests.rs b/integration-tests/ahm/src/tests.rs index 51188b65cd..8c752de508 100644 --- a/integration-tests/ahm/src/tests.rs +++ b/integration-tests/ahm/src/tests.rs @@ -61,7 +61,7 @@ async fn account_migration_works() { dmps.extend(new_dmps); if RcMigrationStage::::get() == - pallet_rc_migrator::MigrationStage::PreimageMigrationDone + pallet_rc_migrator::MigrationStage::AllMigrationsDone { log::info!("Migration done"); break (dmps, pre_check_payload); diff --git a/pallets/ah-migrator/src/lib.rs b/pallets/ah-migrator/src/lib.rs index 0b4554e30d..383abada74 100644 --- a/pallets/ah-migrator/src/lib.rs +++ b/pallets/ah-migrator/src/lib.rs @@ -163,6 +163,18 @@ pub mod pallet { /// How many preimage chunks failed to integrate. count_bad: u32, }, + /// We received a batch of `RcPreimageRequestStatus` that we are going to integrate. + PreimageRequestStatusBatchReceived { + /// How many preimage request status are in the batch. + count: u32, + }, + /// We processed a batch of `RcPreimageRequestStatus` that we received. + PreimageRequestStatusBatchProcessed { + /// How many preimage request status were successfully integrated. + count_good: u32, + /// How many preimage request status failed to integrate. + count_bad: u32, + }, } #[pallet::pallet] @@ -235,6 +247,16 @@ pub mod pallet { Self::do_receive_preimage_chunks(chunks).map_err(Into::into) } + + #[pallet::call_index(5)] + pub fn receive_preimage_request_status( + origin: OriginFor, + request_status: Vec>, + ) -> DispatchResult { + ensure_root(origin)?; + + Self::do_receive_preimage_request_statuses(request_status).map_err(Into::into) + } } #[pallet::hooks] diff --git a/pallets/ah-migrator/src/preimage.rs b/pallets/ah-migrator/src/preimage.rs index 163c7d0935..e66ff95141 100644 --- a/pallets/ah-migrator/src/preimage.rs +++ b/pallets/ah-migrator/src/preimage.rs @@ -16,7 +16,8 @@ // limitations under the License. use crate::{types::*, *}; -use pallet_rc_migrator::preimage::alias; +use frame_support::traits::{Consideration, Footprint}; +use pallet_rc_migrator::preimage::{chunks::*, *}; impl Pallet { pub fn do_receive_preimage_chunks(chunks: Vec) -> Result<(), Error> { @@ -69,14 +70,8 @@ impl Pallet { return Err(Error::::TODO); } - let preimage: BoundedVec< - u8, - ConstU32<{ pallet_rc_migrator::preimage::CHUNK_SIZE }>, - > = chunk.chunk_bytes; - debug_assert!( - pallet_rc_migrator::preimage::CHUNK_SIZE <= - pallet_rc_migrator::preimage::alias::MAX_SIZE - ); + let preimage: BoundedVec> = chunk.chunk_bytes; + debug_assert!(CHUNK_SIZE <= pallet_rc_migrator::preimage::alias::MAX_SIZE); let bounded_preimage: BoundedVec< u8, ConstU32<{ pallet_rc_migrator::preimage::alias::MAX_SIZE }>, @@ -92,4 +87,83 @@ impl Pallet { Ok(()) } + + pub fn do_receive_preimage_request_statuses( + request_status: Vec>, + ) -> Result<(), Error> { + Self::deposit_event(Event::PreimageRequestStatusBatchReceived { + count: request_status.len() as u32, + }); + log::info!(target: LOG_TARGET, "Integrating {} preimage request status", request_status.len()); + let (mut count_good, mut count_bad) = (0, 0); + + for request_status in request_status { + match Self::do_receive_preimage_request_status(request_status) { + Ok(()) => count_good += 1, + Err(e) => { + count_bad += 1; + log::error!(target: LOG_TARGET, "Error while integrating preimage request status: {:?}", e); + }, + } + } + + Self::deposit_event(Event::PreimageRequestStatusBatchProcessed { count_good, count_bad }); + Ok(()) + } + + pub fn do_receive_preimage_request_status( + mut request_status: RcPreimageRequestStatusOf, + ) -> Result<(), Error> { + if alias::RequestStatusFor::::contains_key(&request_status.hash) { + log::warn!(target: LOG_TARGET, "Request status already migrated: {:?}", request_status.hash); + return Ok(()); + } + + let new_ticket = match request_status.request_status { + alias::RequestStatus::Unrequested { ticket: (ref who, ref ticket), len } => { + let fp = Footprint::from_parts(1, len as usize); + ticket.clone().update(&who, fp).ok() + }, + alias::RequestStatus::Requested { + maybe_ticket: Some((ref who, ref ticket)), + maybe_len: Some(len), + .. + } => { + let fp = Footprint::from_parts(1, len as usize); + ticket.clone().update(&who, fp).ok() + }, + alias::RequestStatus::Requested { maybe_ticket: Some(_), maybe_len: None, .. } => { + defensive!("Ticket cannot be re-evaluated"); + // I think this is unreachable, but not exactly sure. Either way, nothing that we + // could do about it. + None + }, + _ => None, + }; + + let new_request_status = match (new_ticket, request_status.request_status.clone()) { + ( + Some(new_ticket), + alias::RequestStatus::Unrequested { ticket: (who, ref mut ticket), len }, + ) => alias::RequestStatus::Unrequested { ticket: (who, new_ticket), len }, + ( + Some(new_ticket), + alias::RequestStatus::Requested { + maybe_ticket: Some((who, ref mut ticket)), + maybe_len: Some(len), + count: count, + }, + ) => alias::RequestStatus::Requested { + maybe_ticket: Some((who, new_ticket)), + maybe_len: Some(len), + count, + }, + _ => request_status.request_status, + }; + + alias::RequestStatusFor::::insert(&request_status.hash, &new_request_status); + log::debug!(target: LOG_TARGET, "Integrating preimage request status: {:?}", new_request_status); + + Ok(()) + } } diff --git a/pallets/rc-migrator/src/lib.rs b/pallets/rc-migrator/src/lib.rs index 69b965e21c..eb2505e998 100644 --- a/pallets/rc-migrator/src/lib.rs +++ b/pallets/rc-migrator/src/lib.rs @@ -64,7 +64,7 @@ use weights::WeightInfo; use xcm::prelude::*; use multisig::MultisigMigrator; -use preimage::PreimageChunkMigrator; +use preimage::{PreimageChunkMigrator, PreimageRequestStatusMigrator}; use proxy::*; use types::PalletMigration; @@ -108,10 +108,15 @@ pub enum MigrationStage { PreimageMigrationInit, PreimageMigrationChunksOngoing { // TODO type - last_key: Option<(Option<(H256, u32)>, u32)>, + last_key: Option<((H256, u32), u32)>, }, PreimageMigrationChunksDone, + PreimageMigrationRequestStatusOngoing { + next_key: Option, + }, + PreimageMigrationRequestStatusDone, PreimageMigrationDone, + AllMigrationsDone, } type AccountInfoFor = @@ -388,7 +393,7 @@ pub mod pallet { match res { Ok(None) => { - Self::transition(MigrationStage::PreimageMigrationDone); + Self::transition(MigrationStage::PreimageMigrationChunksDone); }, Ok(Some(last_key)) => { Self::transition(MigrationStage::PreimageMigrationChunksOngoing { @@ -402,11 +407,45 @@ pub mod pallet { } }, MigrationStage::PreimageMigrationChunksDone => { + Self::transition(MigrationStage::PreimageMigrationRequestStatusOngoing { + next_key: None, + }); + }, + MigrationStage::PreimageMigrationRequestStatusOngoing { next_key } => { + let res = with_transaction_opaque_err::, Error, _>(|| { + TransactionOutcome::Commit( + PreimageRequestStatusMigrator::::migrate_many( + next_key, + &mut weight_counter, + ), + ) + }) + .expect("Always returning Ok; qed"); + + match res { + Ok(None) => { + Self::transition(MigrationStage::PreimageMigrationRequestStatusDone); + }, + Ok(Some(next_key)) => { + Self::transition( + MigrationStage::PreimageMigrationRequestStatusOngoing { + next_key: Some(next_key), + }, + ); + }, + e => { + log::error!(target: LOG_TARGET, "Error while migrating preimage request status: {:?}", e); + defensive!("Error while migrating preimage request status"); + }, + } + }, + MigrationStage::PreimageMigrationRequestStatusDone => { Self::transition(MigrationStage::PreimageMigrationDone); }, MigrationStage::PreimageMigrationDone => { - unimplemented!() + Self::transition(MigrationStage::AllMigrationsDone); }, + MigrationStage::AllMigrationsDone => (), }; weight_counter.consumed() diff --git a/pallets/rc-migrator/src/preimage.rs b/pallets/rc-migrator/src/preimage.rs deleted file mode 100644 index 3f232c70d1..0000000000 --- a/pallets/rc-migrator/src/preimage.rs +++ /dev/null @@ -1,246 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![doc = include_str!("multisig.md")] - -use crate::{types::*, *}; -use sp_runtime::traits::{BlakeTwo256, Hash}; - -pub mod alias { - use super::*; - - use frame_support::{traits::Currency, Identity}; - use sp_core::ConstU32; - - pub const MAX_SIZE: u32 = 4 * 1024 * 1024; - - /// A type to note whether a preimage is owned by a user or the system. - // Copied from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L67-L77 - #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] - pub enum OldRequestStatus { - /// The associated preimage has not yet been requested by the system. The given deposit (if - /// some) is being held until either it becomes requested or the user retracts the - /// preimage. - Unrequested { deposit: (AccountId, Balance), len: u32 }, - /// There are a non-zero number of outstanding requests for this hash by this chain. If - /// there is a preimage registered, then `len` is `Some` and it may be removed iff this - /// counter becomes zero. - Requested { deposit: Option<(AccountId, Balance)>, count: u32, len: Option }, - } - - /// A type to note whether a preimage is owned by a user or the system. - // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L79-L89 - #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] - pub enum RequestStatus { - /// The associated preimage has not yet been requested by the system. The given deposit (if - /// some) is being held until either it becomes requested or the user retracts the - /// preimage. - Unrequested { ticket: (AccountId, Ticket), len: u32 }, - /// There are a non-zero number of outstanding requests for this hash by this chain. If - /// there is a preimage registered, then `len` is `Some` and it may be removed iff this - /// counter becomes zero. - Requested { maybe_ticket: Option<(AccountId, Ticket)>, count: u32, maybe_len: Option }, - } - - // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L91-L93 - type BalanceOf = <::Currency as Currency< - ::AccountId, - >>::Balance; - type TicketOf = ::Consideration; - - // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L173-L185 - #[deprecated = "RequestStatusFor"] - #[frame_support::storage_alias(pallet_name)] - pub type StatusFor = StorageMap< - pallet_preimage::Pallet, - Identity, - H256, - OldRequestStatus<::AccountId, BalanceOf>, - >; - - #[frame_support::storage_alias(pallet_name)] - pub type RequestStatusFor = StorageMap< - pallet_preimage::Pallet, - Identity, - H256, - RequestStatus<::AccountId, TicketOf>, - >; - - #[frame_support::storage_alias(pallet_name)] - pub type PreimageFor = StorageMap< - pallet_preimage::Pallet, - Identity, - (H256, u32), - BoundedVec>, - >; -} - -pub const CHUNK_SIZE: u32 = 49_900; // about 50KiB - -/// A chunk of a preimage that was migrated out of the Relay and can be integrated into AH. -#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] -pub struct RcPreimageChunk { - /// The hash of the original preimage. - pub preimage_hash: H256, - /// The length of the original preimage. - pub preimage_len: u32, - /// Where this chunk starts in the original preimage. - pub chunk_byte_offset: u32, - /// A chunk of the original preimage. - pub chunk_bytes: BoundedVec>, -} - -/// An entry of the `RequestStatusFor` storage map. -#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] -pub struct RcPreimageRequestStatus { - /// The hash of the original preimage. - pub hash: H256, - /// The request status of the original preimage. - pub request_status: alias::RequestStatus, -} - -/// An entry of the `StatusFor` storage map. Should only be used to unreserve funds on AH. -#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] -pub struct RcPreimageLegacyStatus { - /// The hash of the original preimage. - pub hash: H256, - /// The request status of the original preimage. - pub request_status: alias::OldRequestStatus, -} - -pub struct PreimageChunkMigrator { - _phantom: PhantomData, -} - -impl PalletMigration for PreimageChunkMigrator { - type Key = (Option<(H256, u32)>, u32); - type Error = Error; - - fn migrate_many( - mut last_key: Option, - weight_counter: &mut WeightMeter, - ) -> Result, Self::Error> { - let (key_iter, mut last_offset) = match last_key { - None => (alias::PreimageFor::::iter_keys(), 0), - Some((None, offset)) => (alias::PreimageFor::::iter_keys(), offset), - Some((Some((hash, len)), offset)) => ( - alias::PreimageFor::::iter_keys_from(alias::PreimageFor::::hashed_key_for( - &(hash, len), - )), - offset, - ), - }; - - let mut batch = Vec::new(); - let mut current_key = None; - - for kv in key_iter { - // If we're starting a new preimage, reset offset - if current_key.as_ref() != Some(&kv) { - current_key = Some(kv.clone()); - // Reset offset unless we're resuming this specific hash from a previous attempt - let should_reset = - last_key.as_ref().and_then(|k| k.0.as_ref()).map_or(true, |h| h != &kv); - - if should_reset { - last_offset = 0; - } - } - - // Get the full preimage data once - let full_data = alias::PreimageFor::::get(&kv).unwrap_or_default(); - - // Process chunks while there's still data to process - while last_offset < kv.1 { - // Calculate how many bytes remain to be processed - let remaining_bytes = kv.1.saturating_sub(last_offset); - let chunk_size = remaining_bytes.min(CHUNK_SIZE); - - // Extract the chunk - let chunk_bytes: Vec = full_data - .iter() - .skip(last_offset as usize) - .take(chunk_size as usize) - .cloned() - .collect(); - - let bounded_chunk_bytes = BoundedVec::try_from(chunk_bytes) - .expect("Chunk size is bounded by CHUNK_SIZE; qed"); - debug_assert!(bounded_chunk_bytes.len() == chunk_size as usize); - - batch.push(RcPreimageChunk { - preimage_hash: kv.0, - preimage_len: kv.1, - chunk_byte_offset: last_offset, - chunk_bytes: bounded_chunk_bytes, - }); - - log::debug!( - target: LOG_TARGET, - "Processed preimage chunk {:?} at offset {}", - kv, - last_offset - ); - - last_offset += chunk_size; - - // Return after processing 10 chunks, saving our progress - if batch.len() >= 10 { - Pallet::::send_chunked_xcm(batch, |batch| { - types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } - })?; - return Ok(Some((Some(kv), last_offset))); - } - } - - // After finishing a preimage, update last_key and reset offset - last_key = Some((Some(kv), 0)); - last_offset = 0; - } - - // Send any remaining batch before finishing - if !batch.is_empty() { - Pallet::::send_chunked_xcm(batch, |batch| { - types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } - })?; - } - - Ok(None) // No more preimages to process - } -} - -impl PalletMigrationChecks for PreimageChunkMigrator { - type Payload = Vec<(H256, u32)>; - - fn pre_check() -> Self::Payload { - alias::PreimageFor::::iter_keys().collect() - } - - fn post_check(keys: Self::Payload) { - // Check that all keys are inserted - for (hash, len) in keys { - assert!(alias::PreimageFor::::contains_key(&(hash, len))); - } - // Integrity check that all preimages have the correct hash and length - for (hash, len) in alias::PreimageFor::::iter_keys() { - let preimage = alias::PreimageFor::::get(&(hash, len)).expect("Storage corrupted"); - - assert_eq!(preimage.len(), len as usize); - assert_eq!(BlakeTwo256::hash(preimage.as_slice()), hash); - } - } -} diff --git a/pallets/rc-migrator/src/preimage/chunks.rs b/pallets/rc-migrator/src/preimage/chunks.rs new file mode 100644 index 0000000000..9e25e4f18a --- /dev/null +++ b/pallets/rc-migrator/src/preimage/chunks.rs @@ -0,0 +1,162 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{preimage::*, types::*, *}; + +pub const CHUNK_SIZE: u32 = 49_900; // about 50KiB + +/// A chunk of a preimage that was migrated out of the Relay and can be integrated into AH. +#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] +pub struct RcPreimageChunk { + /// The hash of the original preimage. + pub preimage_hash: H256, + /// The length of the original preimage. + pub preimage_len: u32, + /// Where this chunk starts in the original preimage. + pub chunk_byte_offset: u32, + /// A chunk of the original preimage. + pub chunk_bytes: BoundedVec>, +} + +pub struct PreimageChunkMigrator { + _phantom: PhantomData, +} + +impl PalletMigration for PreimageChunkMigrator { + type Key = ((H256, u32), u32); + type Error = Error; + + // The `next_key` is the next key that we will migrate. Not the last one that we migrated. + // This makes the code simpler. + fn migrate_many( + mut next_key: Option, + weight_counter: &mut WeightMeter, + ) -> Result, Self::Error> { + let mut batch = Vec::new(); + + let last_key = loop { + let (next_key_inner, mut last_offset) = match next_key { + None => { + let Some(next_key) = Self::next_key(None) else { + // No more preimages + break None; + }; + (next_key, 0) + }, + Some(((hash, len), offset)) if offset < len => ((hash, len), offset), + Some(((hash, len), offset)) => { + // Get the next key + let Some(next_key) = Self::next_key(Some((hash, len))) else { + break None; + }; + (next_key, 0) + }, + }; + // Load the preimage + let Some(preimage) = alias::PreimageFor::::get(&next_key_inner) else { + defensive!("Storage corruption"); + next_key = Self::next_key(Some(next_key_inner)).map(|(hash, len)| ((hash, len), 0)); + continue; + }; + debug_assert!(last_offset < preimage.len() as u32); + + // Extract the chunk + let chunk_bytes: Vec = preimage + .iter() + .skip(last_offset as usize) + .take(CHUNK_SIZE as usize) + .cloned() + .collect(); + debug_assert!(chunk_bytes.len() > 0); + + let Ok(bounded_chunk) = BoundedVec::try_from(chunk_bytes.clone()).defensive() else { + defensive!("Unreachable"); + next_key = Self::next_key(Some(next_key_inner)).map(|(hash, len)| ((hash, len), 0)); + continue; + }; + + batch.push(RcPreimageChunk { + preimage_hash: next_key_inner.0, + preimage_len: next_key_inner.1, + chunk_byte_offset: last_offset, + chunk_bytes: bounded_chunk, + }); + + last_offset += chunk_bytes.len() as u32; + log::debug!( + target: LOG_TARGET, + "Exported preimage chunk {:?} until offset {}", + next_key_inner, + last_offset + ); + + // set the offset of the next_key + next_key = Some((next_key_inner, last_offset)); + + // TODO weight tracking + if batch.len() >= 10 { + break next_key; + } + }; + + if last_key.is_none() { + log::info!("No more preimages"); + } + + if !batch.is_empty() { + Pallet::::send_chunked_xcm(batch, |batch| { + types::AhMigratorCall::::ReceivePreimageChunks { chunks: batch } + })?; + } + + Ok(last_key) + } +} + +impl PreimageChunkMigrator { + fn next_key(key: Option<(H256, u32)>) -> Option<(H256, u32)> { + match key { + None => alias::PreimageFor::::iter_keys().next(), + Some((hash, len)) => alias::PreimageFor::::iter_keys_from( + alias::PreimageFor::::hashed_key_for(&(hash, len)), + ) + .next(), + } + } +} + +impl PalletMigrationChecks for PreimageChunkMigrator { + type Payload = Vec<(H256, u32)>; + + fn pre_check() -> Self::Payload { + alias::PreimageFor::::iter_keys().collect() + } + + fn post_check(keys: Self::Payload) { + // Check that all keys are inserted + for (hash, len) in keys { + assert!(alias::PreimageFor::::contains_key(&(hash, len))); + } + // Integrity check that all preimages have the correct hash and length + for (hash, len) in alias::PreimageFor::::iter_keys() { + let preimage = alias::PreimageFor::::get(&(hash, len)).expect("Storage corrupted"); + + assert_eq!(preimage.len(), len as usize); + assert_eq!(BlakeTwo256::hash(preimage.as_slice()), hash); + } + } +} diff --git a/pallets/rc-migrator/src/preimage/mod.rs b/pallets/rc-migrator/src/preimage/mod.rs new file mode 100644 index 0000000000..cb5abcb887 --- /dev/null +++ b/pallets/rc-migrator/src/preimage/mod.rs @@ -0,0 +1,105 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc = include_str!("preimage.md")] + +pub mod chunks; +pub mod request_status; + +pub use chunks::{PreimageChunkMigrator, RcPreimageChunk}; +pub use request_status::{PreimageRequestStatusMigrator, RcPreimageRequestStatusOf}; + +use crate::{types::*, *}; +use sp_runtime::traits::{BlakeTwo256, Hash}; + +pub mod alias { + use super::*; + + use frame_support::{traits::Currency, Identity}; + use sp_core::ConstU32; + + pub const MAX_SIZE: u32 = 4 * 1024 * 1024; + + /// A type to note whether a preimage is owned by a user or the system. + // Copied from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L67-L77 + #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] + pub enum OldRequestStatus { + /// The associated preimage has not yet been requested by the system. The given deposit (if + /// some) is being held until either it becomes requested or the user retracts the + /// preimage. + Unrequested { deposit: (AccountId, Balance), len: u32 }, + /// There are a non-zero number of outstanding requests for this hash by this chain. If + /// there is a preimage registered, then `len` is `Some` and it may be removed iff this + /// counter becomes zero. + Requested { deposit: Option<(AccountId, Balance)>, count: u32, len: Option }, + } + + /// A type to note whether a preimage is owned by a user or the system. + // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L79-L89 + #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] + pub enum RequestStatus { + /// The associated preimage has not yet been requested by the system. The given deposit (if + /// some) is being held until either it becomes requested or the user retracts the + /// preimage. + Unrequested { ticket: (AccountId, Ticket), len: u32 }, + /// There are a non-zero number of outstanding requests for this hash by this chain. If + /// there is a preimage registered, then `len` is `Some` and it may be removed iff this + /// counter becomes zero. + Requested { maybe_ticket: Option<(AccountId, Ticket)>, count: u32, maybe_len: Option }, + } + + // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L91-L93 + type BalanceOf = <::Currency as Currency< + ::AccountId, + >>::Balance; + pub type TicketOf = ::Consideration; + + // Coped from https://github.com/paritytech/polkadot-sdk/blob/00946b10ab18331f959f5cbced7c433b6132b1cb/substrate/frame/preimage/src/lib.rs#L173-L185 + #[deprecated = "RequestStatusFor"] + #[frame_support::storage_alias(pallet_name)] + pub type StatusFor = StorageMap< + pallet_preimage::Pallet, + Identity, + H256, + OldRequestStatus<::AccountId, BalanceOf>, + >; + + #[frame_support::storage_alias(pallet_name)] + pub type RequestStatusFor = StorageMap< + pallet_preimage::Pallet, + Identity, + H256, + RequestStatus<::AccountId, TicketOf>, + >; + + #[frame_support::storage_alias(pallet_name)] + pub type PreimageFor = StorageMap< + pallet_preimage::Pallet, + Identity, + (H256, u32), + BoundedVec>, + >; +} + +/// An entry of the `StatusFor` storage map. Should only be used to unreserve funds on AH. +#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] +pub struct RcPreimageLegacyStatus { + /// The hash of the original preimage. + pub hash: H256, + /// The request status of the original preimage. + pub request_status: alias::OldRequestStatus, +} diff --git a/pallets/rc-migrator/src/preimage.md b/pallets/rc-migrator/src/preimage/preimage.md similarity index 100% rename from pallets/rc-migrator/src/preimage.md rename to pallets/rc-migrator/src/preimage/preimage.md diff --git a/pallets/rc-migrator/src/preimage/request_status.rs b/pallets/rc-migrator/src/preimage/request_status.rs new file mode 100644 index 0000000000..18e11402b6 --- /dev/null +++ b/pallets/rc-migrator/src/preimage/request_status.rs @@ -0,0 +1,95 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{preimage::*, types::*, *}; + +/// An entry of the `RequestStatusFor` storage map. +#[derive(Encode, Decode, TypeInfo, Clone, MaxEncodedLen, RuntimeDebug, PartialEq, Eq)] +pub struct RcPreimageRequestStatus { + /// The hash of the original preimage. + pub hash: H256, + /// The request status of the original preimage. + pub request_status: alias::RequestStatus, +} + +pub type RcPreimageRequestStatusOf = + RcPreimageRequestStatus<::AccountId, super::alias::TicketOf>; + +pub struct PreimageRequestStatusMigrator { + _phantom: PhantomData, +} + +impl PalletMigration for PreimageRequestStatusMigrator { + type Key = H256; + type Error = Error; + + fn migrate_many( + mut next_key: Option, + weight_counter: &mut WeightMeter, + ) -> Result, Self::Error> { + let mut batch = Vec::new(); + + let new_next_key = loop { + let next_key_inner = match next_key { + Some(key) => key, + None => { + let Some(key) = Self::next_key(None) else { + break None; + }; + key + }, + }; + + let Some(request_status) = alias::RequestStatusFor::::get(&next_key_inner) else { + defensive!("Storage corruption"); + next_key = Self::next_key(Some(next_key_inner)); + continue; + }; + + batch.push(RcPreimageRequestStatus { hash: next_key_inner, request_status }); + log::debug!(target: LOG_TARGET, "Exported preimage request status for: {:?}", next_key_inner); + + next_key = Self::next_key(Some(next_key_inner)); + + if batch.len() >= 10 { + // TODO weight checking + break next_key; + } + }; + + if !batch.is_empty() { + Pallet::::send_chunked_xcm(batch, |batch| { + types::AhMigratorCall::::ReceivePreimageRequestStatus { request_status: batch } + })?; + } + + Ok(new_next_key) + } +} + +impl PreimageRequestStatusMigrator { + /// Get the next key after the given one or the first one for `None`. + fn next_key(key: Option) -> Option { + match key { + None => alias::RequestStatusFor::::iter_keys().next(), + Some(key) => alias::RequestStatusFor::::iter_keys_from( + alias::RequestStatusFor::::hashed_key_for(&key), + ) + .next(), + } + } +} diff --git a/pallets/rc-migrator/src/types.rs b/pallets/rc-migrator/src/types.rs index e385e2f6fc..3f3edcfc7f 100644 --- a/pallets/rc-migrator/src/types.rs +++ b/pallets/rc-migrator/src/types.rs @@ -40,6 +40,8 @@ pub enum AhMigratorCall { ReceiveProxyAnnouncements { announcements: Vec> }, #[codec(index = 4)] ReceivePreimageChunks { chunks: Vec }, + #[codec(index = 5)] + ReceivePreimageRequestStatus { request_status: Vec> }, } /// Copy of `ParaInfo` type from `paras_registrar` pallet.