From 126f64a91ee0c0333f368fe67bb00d56d68ecb34 Mon Sep 17 00:00:00 2001 From: Javier Viola Date: Tue, 21 Nov 2023 07:38:04 -0300 Subject: [PATCH 01/10] bump zombienet version `v1.3.82` (#2396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Includes: - bump `pjs` modules versions - re-enable test disable in https://github.com/paritytech/polkadot-sdk/pull/2294 --------- Co-authored-by: Bastian Köcher --- .gitlab-ci.yml | 4 ---- .gitlab/pipeline/zombienet.yml | 5 +++++ .gitlab/pipeline/zombienet/substrate.yml | 15 +++++++-------- .../skip-feeless-payment/src/lib.rs | 15 +++++++++------ 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b485ca3317ed..e698aba81d55 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,6 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd NEXTEST_FAILURE_OUTPUT: immediate-final NEXTEST_SUCCESS_OUTPUT: final - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.80" DOCKER_IMAGES_VERSION: "${CI_COMMIT_SHA}" default: @@ -199,9 +198,6 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues -.zombienet-refs: - extends: .build-refs - include: # check jobs - .gitlab/pipeline/check.yml diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml index 64210d6a00ab..fc3fd1256194 100644 --- a/.gitlab/pipeline/zombienet.yml +++ b/.gitlab/pipeline/zombienet.yml @@ -1,3 +1,8 @@ +.zombienet-refs: + extends: .build-refs + variables: + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.82" + include: # substrate tests - .gitlab/pipeline/zombienet/substrate.yml diff --git a/.gitlab/pipeline/zombienet/substrate.yml b/.gitlab/pipeline/zombienet/substrate.yml index 6334e7db9a32..b687576267de 100644 --- a/.gitlab/pipeline/zombienet/substrate.yml +++ b/.gitlab/pipeline/zombienet/substrate.yml @@ -40,14 +40,13 @@ tags: - zombienet-polkadot-integration-test -# Skip this one until PolkadotJS includes `SkipCheckIfFeeless` extension -# zombienet-substrate-0000-block-building: -# extends: -# - .zombienet-substrate-common -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh -# --local-dir="${LOCAL_DIR}/0000-block-building" -# --test="block-building.zndsl" +zombienet-substrate-0000-block-building: + extends: + - .zombienet-substrate-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/0000-block-building" + --test="block-building.zndsl" zombienet-substrate-0001-basic-warp-sync: extends: diff --git a/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs b/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs index 923c7e7ebc29..55afaaf93d78 100644 --- a/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs +++ b/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs @@ -77,9 +77,9 @@ pub mod pallet { /// A [`SignedExtension`] that skips the wrapped extension if the dispatchable is feeless. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] -pub struct SkipCheckIfFeeless(pub S, sp_std::marker::PhantomData); +pub struct SkipCheckIfFeeless(pub S, sp_std::marker::PhantomData); -impl sp_std::fmt::Debug for SkipCheckIfFeeless { +impl sp_std::fmt::Debug for SkipCheckIfFeeless { #[cfg(feature = "std")] fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { write!(f, "SkipCheckIfFeeless<{:?}>", self.0.encode()) @@ -90,9 +90,8 @@ impl sp_std::fmt::Debug for SkipCheckIfFeeless SkipCheckIfFeeless { - /// utility constructor. Used only in client/factory code. - pub fn from(s: S) -> Self { +impl From for SkipCheckIfFeeless { + fn from(s: S) -> Self { Self(s, sp_std::marker::PhantomData) } } @@ -106,7 +105,11 @@ where type Call = S::Call; type AdditionalSigned = S::AdditionalSigned; type Pre = (Self::AccountId, Option<::Pre>); - const IDENTIFIER: &'static str = "SkipCheckIfFeeless"; + // From the outside this extension should be "invisible", because it just extends the wrapped + // extension with an extra check in `pre_dispatch` and `post_dispatch`. Thus, we should forward + // the identifier of the wrapped extension to let wallets see this extension as it would only be + // the wrapped extension itself. + const IDENTIFIER: &'static str = S::IDENTIFIER; fn additional_signed(&self) -> Result { self.0.additional_signed() From 2fd8c51ebcfcf7d10cab2a2827211583418a90a0 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:51:23 +0100 Subject: [PATCH 02/10] `chain-spec-builder`: cleanup (#2174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR removes: - `New`, `Generate`, `Edit` commands, - `kitchensink` dependency from the `chain-spec-builder` util. New `convert-to-raw`, `update-code` commands were added. Additionally renames the `runtime` command (which was added in #1256) to `create`. --------- Co-authored-by: Sebastian Kunert Co-authored-by: Bastian Köcher Co-authored-by: command-bot <> --- Cargo.lock | 7 - .../bin/utils/chain-spec-builder/Cargo.toml | 7 - .../bin/utils/chain-spec-builder/bin/main.rs | 127 ++------ .../bin/utils/chain-spec-builder/src/lib.rs | 295 +++++------------- 4 files changed, 116 insertions(+), 320 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2d655e6f41c..697f232f9719 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18315,18 +18315,11 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" name = "staging-chain-spec-builder" version = "2.0.0" dependencies = [ - "ansi_term", "clap 4.4.6", - "kitchensink-runtime", "log", - "rand 0.8.5", "sc-chain-spec", - "sc-keystore", "serde_json", - "sp-core", - "sp-keystore", "sp-tracing 10.0.0", - "staging-node-cli", ] [[package]] diff --git a/substrate/bin/utils/chain-spec-builder/Cargo.toml b/substrate/bin/utils/chain-spec-builder/Cargo.toml index 0b373b8e9247..f587989e0039 100644 --- a/substrate/bin/utils/chain-spec-builder/Cargo.toml +++ b/substrate/bin/utils/chain-spec-builder/Cargo.toml @@ -20,15 +20,8 @@ name = "chain-spec-builder" crate-type = ["rlib"] [dependencies] -ansi_term = "0.12.1" clap = { version = "4.4.6", features = ["derive"] } -rand = "0.8" -kitchensink-runtime = { version = "3.0.0-dev", path = "../../node/runtime" } log = "0.4.17" -node-cli = { package = "staging-node-cli", path = "../../node/cli" } sc-chain-spec = { path = "../../../client/chain-spec" } -sc-keystore = { path = "../../../client/keystore" } serde_json = "1.0.108" -sp-core = { path = "../../../primitives/core" } -sp-keystore = { path = "../../../primitives/keystore" } sp-tracing = { version = "10.0.0", path = "../../../primitives/tracing" } diff --git a/substrate/bin/utils/chain-spec-builder/bin/main.rs b/substrate/bin/utils/chain-spec-builder/bin/main.rs index 83892afd6ace..986293179a91 100644 --- a/substrate/bin/utils/chain-spec-builder/bin/main.rs +++ b/substrate/bin/utils/chain-spec-builder/bin/main.rs @@ -17,14 +17,11 @@ // along with this program. If not, see . use chain_spec_builder::{ - generate_authority_keys_and_store, generate_chain_spec, generate_chain_spec_for_runtime, - print_seeds, ChainSpecBuilder, ChainSpecBuilderCmd, EditCmd, GenerateCmd, NewCmd, VerifyCmd, + generate_chain_spec_for_runtime, ChainSpecBuilder, ChainSpecBuilderCmd, ConvertToRawCmd, + UpdateCodeCmd, VerifyCmd, }; use clap::Parser; -use node_cli::chain_spec; -use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; use sc_chain_spec::{update_code_in_json_chain_spec, GenericChainSpec}; -use sp_core::{crypto::Ss58Codec, sr25519}; use staging_chain_spec_builder as chain_spec_builder; use std::fs; @@ -32,110 +29,48 @@ fn main() -> Result<(), String> { sp_tracing::try_init_simple(); let builder = ChainSpecBuilder::parse(); - #[cfg(build_type = "debug")] - if matches!(builder.command, ChainSpecBuilderCmd::Generate(_) | ChainSpecBuilderCmd::New(_)) { - println!( - "The chain spec builder builds a chain specification that includes a Substrate runtime \ - compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \ - the chain spec builder binary in `--release` mode.\n", - ); - } - let chain_spec_path = builder.chain_spec_path.to_path_buf(); - let mut write_chain_spec = true; - - let chain_spec_json = match builder.command { - ChainSpecBuilderCmd::Generate(GenerateCmd { - authorities, - nominators, - endowed, - keystore_path, - }) => { - let authorities = authorities.max(1); - let rand_str = || -> String { - OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect() - }; - - let authority_seeds = (0..authorities).map(|_| rand_str()).collect::>(); - let nominator_seeds = (0..nominators).map(|_| rand_str()).collect::>(); - let endowed_seeds = (0..endowed).map(|_| rand_str()).collect::>(); - let sudo_seed = rand_str(); - - print_seeds(&authority_seeds, &nominator_seeds, &endowed_seeds, &sudo_seed); - - if let Some(keystore_path) = keystore_path { - generate_authority_keys_and_store(&authority_seeds, &keystore_path)?; - } - - let nominator_accounts = nominator_seeds - .into_iter() - .map(|seed| { - chain_spec::get_account_id_from_seed::(&seed).to_ss58check() - }) - .collect(); - - let endowed_accounts = endowed_seeds - .into_iter() - .map(|seed| { - chain_spec::get_account_id_from_seed::(&seed).to_ss58check() - }) - .collect(); - let sudo_account = - chain_spec::get_account_id_from_seed::(&sudo_seed).to_ss58check(); - - generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account) + match builder.command { + ChainSpecBuilderCmd::Create(cmd) => { + let chain_spec_json = generate_chain_spec_for_runtime(&cmd)?; + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string())?; }, - ChainSpecBuilderCmd::New(NewCmd { - authority_seeds, - nominator_accounts, - endowed_accounts, - sudo_account, - }) => - generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account), - ChainSpecBuilderCmd::Runtime(cmd) => generate_chain_spec_for_runtime(&cmd), - ChainSpecBuilderCmd::Edit(EditCmd { + ChainSpecBuilderCmd::UpdateCode(UpdateCodeCmd { ref input_chain_spec, ref runtime_wasm_path, - convert_to_raw, }) => { let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; let mut chain_spec_json = - serde_json::from_str::(&chain_spec.as_json(convert_to_raw)?) + serde_json::from_str::(&chain_spec.as_json(false)?) .map_err(|e| format!("Conversion to json failed: {e}"))?; - if let Some(path) = runtime_wasm_path { - update_code_in_json_chain_spec( - &mut chain_spec_json, - &fs::read(path.as_path()) - .map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..], - ); - } - - serde_json::to_string_pretty(&chain_spec_json) - .map_err(|e| format!("to pretty failed: {e}")) + update_code_in_json_chain_spec( + &mut chain_spec_json, + &fs::read(runtime_wasm_path.as_path()) + .map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..], + ); + + let chain_spec_json = serde_json::to_string_pretty(&chain_spec_json) + .map_err(|e| format!("to pretty failed: {e}"))?; + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string())?; }, - ChainSpecBuilderCmd::Verify(VerifyCmd { ref input_chain_spec, ref runtime_wasm_path }) => { - write_chain_spec = false; + ChainSpecBuilderCmd::ConvertToRaw(ConvertToRawCmd { ref input_chain_spec }) => { let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; - let mut chain_spec_json = + + let chain_spec_json = serde_json::from_str::(&chain_spec.as_json(true)?) .map_err(|e| format!("Conversion to json failed: {e}"))?; - if let Some(path) = runtime_wasm_path { - update_code_in_json_chain_spec( - &mut chain_spec_json, - &fs::read(path.as_path()) - .map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..], - ); - }; - serde_json::to_string_pretty(&chain_spec_json) - .map_err(|e| format!("to pretty failed: {e}")) - }, - }?; - if write_chain_spec { - fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string()) - } else { - Ok(()) - } + let chain_spec_json = serde_json::to_string_pretty(&chain_spec_json) + .map_err(|e| format!("Conversion to pretty failed: {e}"))?; + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string())?; + }, + ChainSpecBuilderCmd::Verify(VerifyCmd { ref input_chain_spec }) => { + let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; + let _ = serde_json::from_str::(&chain_spec.as_json(true)?) + .map_err(|e| format!("Conversion to json failed: {e}"))?; + }, + }; + Ok(()) } diff --git a/substrate/bin/utils/chain-spec-builder/src/lib.rs b/substrate/bin/utils/chain-spec-builder/src/lib.rs index 6f21b68c3684..60ec0f1a656b 100644 --- a/substrate/bin/utils/chain-spec-builder/src/lib.rs +++ b/substrate/bin/utils/chain-spec-builder/src/lib.rs @@ -21,31 +21,69 @@ //! A chain-spec is short for `chain-configuration`. See the [`sc-chain-spec`] for more information. //! //! Note that this binary is analogous to the `build-spec` subcommand, contained in typical -//! substrate-based nodes. This particular binary is capable of building a more sophisticated chain -//! specification that can be used with the substrate-node, ie. [`node-cli`]. +//! substrate-based nodes. This particular binary is capable of interacting with +//! [`sp-genesis-builder`] implementation of any provided runtime allowing to build chain-spec JSON +//! files. //! -//! See [`ChainSpecBuilder`] for a list of available commands. +//! See [`ChainSpecBuilderCmd`] for a list of available commands. +//! +//! ## Typical use-cases. +//! ##### Get default config from runtime. +//! +//! Query the default genesis config from the provided `runtime.wasm` and use it in the chain +//! spec. Tool can also store runtime's default genesis config in given file: +//! ```text +//! chain-spec-builder create -r runtime.wasm default /dev/stdout +//! ``` +//! +//! _Note:_ [`GenesisBuilder::create_default_config`][sp-genesis-builder-create] runtime function is called. +//! +//! +//! ##### Generate raw storage chain spec using genesis config patch. +//! +//! Patch the runtime's default genesis config with provided `patch.json` and generate raw +//! storage (`-s`) version of chain spec: +//! ```text +//! chain-spec-builder create -s -r runtime.wasm patch patch.json +//! ``` +//! +//! _Note:_ [`GenesisBuilder::build_config`][sp-genesis-builder-build] runtime function is called. +//! +//! ##### Generate raw storage chain spec using full genesis config. +//! +//! Build the chain spec using provided full genesis config json file. No defaults will be used: +//! ```text +//! chain-spec-builder create -s -r runtime.wasm full full-genesis-config.json +//! ``` +//! +//! _Note_: [`GenesisBuilder::build_config`][sp-genesis-builder-build] runtime function is called. +//! +//! ##### Generate human readable chain spec using provided genesis config patch. +//! ```text +//! chain-spec-builder create -r runtime.wasm patch patch.json +//! ``` +//! +//! ##### Generate human readable chain spec using provided full genesis config. +//! ```text +//! chain-spec-builder create -r runtime.wasm full full-genesis-config.json +//! ``` +//! +//! ##### Extra tools. +//! The `chain-spec-builder` provides also some extra utilities: [`VerifyCmd`], [`ConvertToRawCmd`], [`UpdateCodeCmd`]. //! //! [`sc-chain-spec`]: ../sc_chain_spec/index.html //! [`node-cli`]: ../node_cli/index.html +//! [`sp-genesis-builder`]: ../sp_genesis_builder/index.html +//! [sp-genesis-builder-create]: ../sp_genesis_builder/trait.GenesisBuilder.html#method.create_default_config +//! [sp-genesis-builder-build]: ../sp_genesis_builder/trait.GenesisBuilder.html#method.build_config -use std::{ - fs, - path::{Path, PathBuf}, -}; +use std::{fs, path::PathBuf}; -use ansi_term::Style; use clap::{Parser, Subcommand}; -use sc_chain_spec::GenesisConfigBuilderRuntimeCaller; - -use node_cli::chain_spec::{self, AccountId}; -use sc_keystore::LocalKeystore; +use sc_chain_spec::{GenericChainSpec, GenesisConfigBuilderRuntimeCaller}; use serde_json::Value; -use sp_core::crypto::{ByteArray, Ss58Codec}; -use sp_keystore::KeystorePtr; -/// A utility to easily create a testnet chain spec definition with a given set -/// of authorities and endowed accounts and/or generate random accounts. +/// A utility to easily create a chain spec definition. #[derive(Debug, Parser)] #[command(rename_all = "kebab-case")] pub struct ChainSpecBuilder { @@ -59,70 +97,25 @@ pub struct ChainSpecBuilder { #[derive(Debug, Subcommand)] #[command(rename_all = "kebab-case")] pub enum ChainSpecBuilderCmd { - New(NewCmd), - Generate(GenerateCmd), - Runtime(RuntimeCmd), - Edit(EditCmd), + Create(CreateCmd), Verify(VerifyCmd), -} - -/// Create a new chain spec with the given authorities, endowed and sudo -/// accounts. Only works for kitchen-sink runtime -#[derive(Parser, Debug)] -#[command(rename_all = "kebab-case")] -pub struct NewCmd { - /// Authority key seed. - #[arg(long, short, required = true)] - pub authority_seeds: Vec, - /// Active nominators (SS58 format), each backing a random subset of the aforementioned - /// authorities. - #[arg(long, short, default_value = "0")] - pub nominator_accounts: Vec, - /// Endowed account address (SS58 format). - #[arg(long, short)] - pub endowed_accounts: Vec, - /// Sudo account address (SS58 format). - #[arg(long, short)] - pub sudo_account: String, -} - -/// Create a new chain spec with the given number of authorities and endowed -/// accounts. Random keys will be generated as required. -#[derive(Parser, Debug)] -pub struct GenerateCmd { - /// The number of authorities. - #[arg(long, short)] - pub authorities: usize, - /// The number of nominators backing the aforementioned authorities. - /// - /// Will nominate a random subset of `authorities`. - #[arg(long, short, default_value_t = 0)] - pub nominators: usize, - /// The number of endowed accounts. - #[arg(long, short, default_value_t = 0)] - pub endowed: usize, - /// Path to use when saving generated keystores for each authority. - /// - /// At this path, a new folder will be created for each authority's - /// keystore named `auth-$i` where `i` is the authority index, i.e. - /// `auth-0`, `auth-1`, etc. - #[arg(long, short)] - pub keystore_path: Option, + UpdateCode(UpdateCodeCmd), + ConvertToRaw(ConvertToRawCmd), } /// Create a new chain spec by interacting with the provided runtime wasm blob. #[derive(Parser, Debug)] -pub struct RuntimeCmd { - /// The name of chain +pub struct CreateCmd { + /// The name of chain. #[arg(long, short = 'n', default_value = "Custom")] chain_name: String, - /// The chain id + /// The chain id. #[arg(long, short = 'i', default_value = "custom")] chain_id: String, - /// The path to runtime wasm blob + /// The path to runtime wasm blob. #[arg(long, short)] runtime_wasm_path: PathBuf, - /// Export chainspec as raw storage + /// Export chainspec as raw storage. #[arg(long, short = 's')] raw_storage: bool, /// Verify the genesis config. This silently generates the raw storage from genesis config. Any @@ -144,7 +137,6 @@ enum GenesisBuildAction { #[derive(Parser, Debug, Clone)] struct PatchCmd { /// The path to the runtime genesis config patch. - #[arg(long, short)] patch_path: PathBuf, } @@ -152,7 +144,6 @@ struct PatchCmd { #[derive(Parser, Debug, Clone)] struct FullCmd { /// The path to the full runtime genesis config json file. - #[arg(long, short)] config_path: PathBuf, } @@ -163,161 +154,45 @@ struct FullCmd { struct DefaultCmd { /// If provided stores the default genesis config json file at given path (in addition to /// chain-spec). - #[arg(long, short)] default_config_path: Option, } -/// Edits provided input chain spec. Input can be converted into raw storage chain-spec. The code -/// can be updated with the runtime provided in the command line. +/// Updates the code in the provided input chain spec. +/// +/// The code field of the chain spec will be updated with the runtime provided in the +/// command line. This operation supports both plain and raw formats. #[derive(Parser, Debug, Clone)] -pub struct EditCmd { - /// Chain spec to be edited - #[arg(long, short)] +pub struct UpdateCodeCmd { + /// Chain spec to be updated. pub input_chain_spec: PathBuf, - /// The path to new runtime wasm blob to be stored into chain-spec - #[arg(long, short = 'r')] - pub runtime_wasm_path: Option, - /// Convert genesis spec to raw format - #[arg(long, short = 's')] - pub convert_to_raw: bool, + /// The path to new runtime wasm blob to be stored into chain-spec. + pub runtime_wasm_path: PathBuf, } -/// Verifies provided input chain spec. If the runtime is provided verification is performed against -/// new runtime. +/// Converts the given chain spec into the raw format. #[derive(Parser, Debug, Clone)] -pub struct VerifyCmd { - /// Chain spec to be edited - #[arg(long, short)] +pub struct ConvertToRawCmd { + /// Chain spec to be converted. pub input_chain_spec: PathBuf, - /// The path to new runtime wasm blob to be stored into chain-spec - #[arg(long, short = 'r')] - pub runtime_wasm_path: Option, -} - -/// Generate the chain spec using the given seeds and accounts. -pub fn generate_chain_spec( - authority_seeds: Vec, - nominator_accounts: Vec, - endowed_accounts: Vec, - sudo_account: String, -) -> Result { - let parse_account = |address: String| { - AccountId::from_string(&address) - .map_err(|err| format!("Failed to parse account address: {:?}", err)) - }; - - let nominator_accounts = nominator_accounts - .into_iter() - .map(parse_account) - .collect::, String>>()?; - - let endowed_accounts = endowed_accounts - .into_iter() - .map(parse_account) - .collect::, String>>()?; - - let sudo_account = parse_account(sudo_account)?; - - let authorities = authority_seeds - .iter() - .map(AsRef::as_ref) - .map(chain_spec::authority_keys_from_seed) - .collect::>(); - - chain_spec::ChainSpec::builder(kitchensink_runtime::wasm_binary_unwrap(), Default::default()) - .with_name("Custom") - .with_id("custom") - .with_chain_type(sc_chain_spec::ChainType::Live) - .with_genesis_config_patch(chain_spec::testnet_genesis( - authorities, - nominator_accounts, - sudo_account, - Some(endowed_accounts), - )) - .build() - .as_json(false) -} - -/// Generate the authority keys and store them in the given `keystore_path`. -pub fn generate_authority_keys_and_store( - seeds: &[String], - keystore_path: &Path, -) -> Result<(), String> { - for (n, seed) in seeds.iter().enumerate() { - let keystore: KeystorePtr = - LocalKeystore::open(keystore_path.join(format!("auth-{}", n)), None) - .map_err(|err| err.to_string())? - .into(); - - let (_, _, grandpa, babe, im_online, authority_discovery, mixnet) = - chain_spec::authority_keys_from_seed(seed); - - let insert_key = |key_type, public| { - keystore - .insert(key_type, &format!("//{}", seed), public) - .map_err(|_| format!("Failed to insert key: {}", grandpa)) - }; - - insert_key(sp_core::crypto::key_types::BABE, babe.as_slice())?; - - insert_key(sp_core::crypto::key_types::GRANDPA, grandpa.as_slice())?; - - insert_key(sp_core::crypto::key_types::IM_ONLINE, im_online.as_slice())?; - - insert_key( - sp_core::crypto::key_types::AUTHORITY_DISCOVERY, - authority_discovery.as_slice(), - )?; - - insert_key(sp_core::crypto::key_types::MIXNET, mixnet.as_slice())?; - } - - Ok(()) } -/// Print the given seeds -pub fn print_seeds( - authority_seeds: &[String], - nominator_seeds: &[String], - endowed_seeds: &[String], - sudo_seed: &str, -) { - let header = Style::new().bold().underline(); - let entry = Style::new().bold(); - - println!("{}", header.paint("Authority seeds")); - - for (n, seed) in authority_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("auth-{}:", n)), seed); - } - - println!("{}", header.paint("Nominator seeds")); - - for (n, seed) in nominator_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("nom-{}:", n)), seed); - } - - println!(); - - if !endowed_seeds.is_empty() { - println!("{}", header.paint("Endowed seeds")); - for (n, seed) in endowed_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("endowed-{}:", n)), seed); - } - - println!(); - } - - println!("{}", header.paint("Sudo seed")); - println!("//{}", sudo_seed); +/// Verifies the provided input chain spec. +/// +/// Silently checks if given input chain spec can be converted to raw. It allows to check if all +/// RuntimeGenesisConfig fiels are properly initialized and if the json does not contain invalid +/// fields. +#[derive(Parser, Debug, Clone)] +pub struct VerifyCmd { + /// Chain spec to be verified. + pub input_chain_spec: PathBuf, } -/// Processes `RuntimeCmd` and returns JSON version of `ChainSpec` -pub fn generate_chain_spec_for_runtime(cmd: &RuntimeCmd) -> Result { +/// Processes `CreateCmd` and returns JSON version of `ChainSpec`. +pub fn generate_chain_spec_for_runtime(cmd: &CreateCmd) -> Result { let code = fs::read(cmd.runtime_wasm_path.as_path()) .map_err(|e| format!("wasm blob shall be readable {e}"))?; - let builder = chain_spec::ChainSpec::builder(&code[..], Default::default()) + let builder = GenericChainSpec::<()>::builder(&code[..], Default::default()) .with_name(&cmd.chain_name[..]) .with_id(&cmd.chain_id[..]) .with_chain_type(sc_chain_spec::ChainType::Live); From 06190498a00ff913975fa8853c3c6a94783c4124 Mon Sep 17 00:00:00 2001 From: Julian Eager Date: Tue, 21 Nov 2023 19:10:59 +0800 Subject: [PATCH 03/10] Refactor `ValidationError` (#2406) --- .../node/core/candidate-validation/src/lib.rs | 60 +++++++++++-------- .../core/candidate-validation/src/tests.rs | 40 ++++++------- .../benches/host_prepare_rococo_runtime.rs | 3 +- .../node/core/pvf/common/src/executor_intf.rs | 19 +++--- polkadot/node/core/pvf/src/error.rs | 37 ++++++++---- polkadot/node/core/pvf/src/execute/queue.rs | 14 ++--- polkadot/node/core/pvf/src/host.rs | 39 +++++------- polkadot/node/core/pvf/src/lib.rs | 2 +- polkadot/node/core/pvf/tests/it/main.rs | 8 +-- polkadot/node/core/pvf/tests/it/process.rs | 10 ++-- 10 files changed, 122 insertions(+), 110 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index f5d17af6c689..9f7b17f61299 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -24,8 +24,8 @@ #![warn(missing_docs)] use polkadot_node_core_pvf::{ - InternalValidationError, InvalidCandidate as WasmInvalidCandidate, PrepareError, - PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, + InternalValidationError, InvalidCandidate as WasmInvalidCandidate, PossiblyInvalidError, + PrepareError, PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, }; use polkadot_node_primitives::{ BlockData, InvalidCandidate, PoV, ValidationResult, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT, @@ -642,7 +642,7 @@ async fn validate_candidate_exhaustive( } match result { - Err(ValidationError::InternalError(e)) => { + Err(ValidationError::Internal(e)) => { gum::warn!( target: LOG_TARGET, ?para_id, @@ -651,34 +651,29 @@ async fn validate_candidate_exhaustive( ); Err(ValidationFailed(e.to_string())) }, - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)) => + Err(ValidationError::Invalid(WasmInvalidCandidate::HardTimeout)) => Ok(ValidationResult::Invalid(InvalidCandidate::Timeout)), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::WorkerReportedInvalid(e))) => + Err(ValidationError::Invalid(WasmInvalidCandidate::WorkerReportedInvalid(e))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(e))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)) => + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError( "ambiguous worker death".to_string(), ))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError(err))) => + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(err))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousJobDeath(err))) => + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(format!( "ambiguous job death: {err}" )))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::PrepareError(e))) => { - // In principle if preparation of the `WASM` fails, the current candidate can not be the - // reason for that. So we can't say whether it is invalid or not. In addition, with - // pre-checking enabled only valid runtimes should ever get enacted, so we can be - // reasonably sure that this is some local problem on the current node. However, as this - // particular error *seems* to indicate a deterministic error, we raise a warning. + Err(ValidationError::Preparation(e)) => { gum::warn!( target: LOG_TARGET, ?para_id, ?e, "Deterministic error occurred during preparation (should have been ruled out by pre-checking phase)", ); - Err(ValidationFailed(e)) + Err(ValidationFailed(e.to_string())) }, Ok(res) => if res.head_data.hash() != candidate_receipt.descriptor.para_head { @@ -762,16 +757,31 @@ trait ValidationBackend { } match validation_result { - Err(ValidationError::InvalidCandidate( - WasmInvalidCandidate::AmbiguousWorkerDeath | - WasmInvalidCandidate::AmbiguousJobDeath(_), - )) if num_death_retries_left > 0 => num_death_retries_left -= 1, - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError(_))) - if num_job_error_retries_left > 0 => - num_job_error_retries_left -= 1, - Err(ValidationError::InternalError(_)) if num_internal_retries_left > 0 => - num_internal_retries_left -= 1, - _ => break, + Err(ValidationError::PossiblyInvalid( + PossiblyInvalidError::AmbiguousWorkerDeath | + PossiblyInvalidError::AmbiguousJobDeath(_), + )) => + if num_death_retries_left > 0 { + num_death_retries_left -= 1; + } else { + break; + }, + + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(_))) => + if num_job_error_retries_left > 0 { + num_job_error_retries_left -= 1; + } else { + break; + }, + + Err(ValidationError::Internal(_)) => + if num_internal_retries_left > 0 { + num_internal_retries_left -= 1; + } else { + break; + }, + + Ok(_) | Err(ValidationError::Invalid(_) | ValidationError::Preparation(_)) => break, } // If we got a possibly transient error, retry once after a brief delay, on the diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 5e2585d68735..110785804652 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -480,9 +480,9 @@ fn candidate_validation_bad_return_is_invalid() { let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Err( - ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout), - )), + MockValidateCandidateBackend::with_hardcoded_result(Err(ValidationError::Invalid( + WasmInvalidCandidate::HardTimeout, + ))), validation_data, validation_code, candidate_receipt, @@ -561,7 +561,7 @@ fn candidate_validation_one_ambiguous_error_is_valid() { let v = executor::block_on(validate_candidate_exhaustive( MockValidateCandidateBackend::with_hardcoded_result_list(vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), Ok(validation_result), ]), validation_data.clone(), @@ -602,8 +602,8 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { let v = executor::block_on(validate_candidate_exhaustive( MockValidateCandidateBackend::with_hardcoded_result_list(vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), ]), validation_data, validation_code, @@ -626,7 +626,7 @@ fn candidate_validation_retry_internal_errors() { vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AJD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousJobDeath( + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath( "baz".into(), ))), // Throw another internal error. @@ -644,7 +644,7 @@ fn candidate_validation_dont_retry_internal_errors() { vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), // Throw another internal error. Err(InternalValidationError::HostCommunication("bar".into()).into()), ], @@ -659,11 +659,11 @@ fn candidate_validation_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( PvfExecKind::Approval, vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("foo".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), // Throw another panic error. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("bar".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("bar".into()))), ], ); @@ -676,11 +676,11 @@ fn candidate_validation_dont_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( PvfExecKind::Backing, vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("foo".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), // Throw another panic error. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("bar".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("bar".into()))), ], ); @@ -758,9 +758,9 @@ fn candidate_validation_timeout_is_internal_error() { let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Err( - ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout), - )), + MockValidateCandidateBackend::with_hardcoded_result(Err(ValidationError::Invalid( + WasmInvalidCandidate::HardTimeout, + ))), validation_data, validation_code, candidate_receipt, @@ -852,9 +852,9 @@ fn candidate_validation_code_mismatch_is_invalid() { let (_ctx, _ctx_handle) = test_helpers::make_subsystem_context::(pool.clone()); let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Err( - ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout), - )), + MockValidateCandidateBackend::with_hardcoded_result(Err(ValidationError::Invalid( + WasmInvalidCandidate::HardTimeout, + ))), validation_data, validation_code, candidate_receipt, diff --git a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs index c02a0b595da3..f2551b701c2c 100644 --- a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs +++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs @@ -18,8 +18,7 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion, SamplingMode}; use polkadot_node_core_pvf::{ - start, testing, Config, Metrics, PrepareError, PrepareJobKind, PrepareStats, PvfPrepData, - ValidationHost, + start, testing, Config, Metrics, PrepareError, PrepareJobKind, PvfPrepData, ValidationHost, }; use polkadot_primitives::ExecutorParams; use rococo_runtime::WASM_BINARY; diff --git a/polkadot/node/core/pvf/common/src/executor_intf.rs b/polkadot/node/core/pvf/common/src/executor_intf.rs index 3a1d3ac1ba07..4bafc3f4291a 100644 --- a/polkadot/node/core/pvf/common/src/executor_intf.rs +++ b/polkadot/node/core/pvf/common/src/executor_intf.rs @@ -140,8 +140,7 @@ pub unsafe fn create_runtime_from_artifact_bytes( executor_params: &ExecutorParams, ) -> Result { let mut config = DEFAULT_CONFIG.clone(); - config.semantics = - params_to_wasmtime_semantics(executor_params).map_err(|err| WasmError::Other(err))?; + config.semantics = params_to_wasmtime_semantics(executor_params); sc_executor_wasmtime::create_runtime_from_artifact_bytes::( compiled_artifact_blob, @@ -149,13 +148,12 @@ pub unsafe fn create_runtime_from_artifact_bytes( ) } -pub fn params_to_wasmtime_semantics(par: &ExecutorParams) -> Result { +pub fn params_to_wasmtime_semantics(par: &ExecutorParams) -> Semantics { let mut sem = DEFAULT_CONFIG.semantics.clone(); - let mut stack_limit = if let Some(stack_limit) = sem.deterministic_stack_limit.clone() { - stack_limit - } else { - return Err("No default stack limit set".to_owned()) - }; + let mut stack_limit = sem + .deterministic_stack_limit + .expect("There is a comment to not change the default stack limit; it should always be available; qed") + .clone(); for p in par.iter() { match p { @@ -172,7 +170,7 @@ pub fn params_to_wasmtime_semantics(par: &ExecutorParams) -> Result Result, sc_executor_common::error::WasmError> { - let semantics = params_to_wasmtime_semantics(executor_params) - .map_err(|e| sc_executor_common::error::WasmError::Other(e))?; + let semantics = params_to_wasmtime_semantics(executor_params); sc_executor_wasmtime::prepare_runtime_artifact(blob, &semantics) } diff --git a/polkadot/node/core/pvf/src/error.rs b/polkadot/node/core/pvf/src/error.rs index 7fdb8c56ec92..442443f326e9 100644 --- a/polkadot/node/core/pvf/src/error.rs +++ b/polkadot/node/core/pvf/src/error.rs @@ -19,31 +19,44 @@ use polkadot_node_core_pvf_common::error::{InternalValidationError, PrepareError /// A error raised during validation of the candidate. #[derive(Debug, Clone)] pub enum ValidationError { - /// The error was raised because the candidate is invalid. + /// Deterministic preparation issue. In practice, most of the problems should be caught by + /// prechecking, so this may be a sign of internal conditions. /// - /// Whenever we are unsure if the error was due to the candidate or not, we must vote invalid. - InvalidCandidate(InvalidCandidate), - /// Some internal error occurred. - InternalError(InternalValidationError), + /// In principle if preparation of the `WASM` fails, the current candidate cannot be the + /// reason for that. So we can't say whether it is invalid or not. In addition, with + /// pre-checking enabled only valid runtimes should ever get enacted, so we can be + /// reasonably sure that this is some local problem on the current node. However, as this + /// particular error *seems* to indicate a deterministic error, we raise a warning. + Preparation(PrepareError), + /// The error was raised because the candidate is invalid. Should vote against. + Invalid(InvalidCandidate), + /// Possibly transient issue that may resolve after retries. Should vote against when retries + /// fail. + PossiblyInvalid(PossiblyInvalidError), + /// Preparation or execution issue caused by an internal condition. Should not vote against. + Internal(InternalValidationError), } /// A description of an error raised during executing a PVF and can be attributed to the combination /// of the candidate [`polkadot_parachain_primitives::primitives::ValidationParams`] and the PVF. #[derive(Debug, Clone)] pub enum InvalidCandidate { - /// PVF preparation ended up with a deterministic error. - PrepareError(String), /// The candidate is reported to be invalid by the execution worker. The string contains the /// error message. WorkerReportedInvalid(String), + /// PVF execution (compilation is not included) took more time than was allotted. + HardTimeout, +} + +/// Possibly transient issue that may resolve after retries. +#[derive(Debug, Clone)] +pub enum PossiblyInvalidError { /// The worker process (not the job) has died during validation of a candidate. /// /// It's unlikely that this is caused by malicious code since workers spawn separate job /// processes, and those job processes are sandboxed. But, it is possible. We retry in this /// case, and if the error persists, we assume it's caused by the candidate and vote against. AmbiguousWorkerDeath, - /// PVF execution (compilation is not included) took more time than was allotted. - HardTimeout, /// The job process (not the worker) has died for one of the following reasons: /// /// (a) A seccomp violation occurred, most likely due to an attempt by malicious code to @@ -68,7 +81,7 @@ pub enum InvalidCandidate { impl From for ValidationError { fn from(error: InternalValidationError) -> Self { - Self::InternalError(error) + Self::Internal(error) } } @@ -77,9 +90,9 @@ impl From for ValidationError { // Here we need to classify the errors into two errors: deterministic and non-deterministic. // See [`PrepareError::is_deterministic`]. if error.is_deterministic() { - Self::InvalidCandidate(InvalidCandidate::PrepareError(error.to_string())) + Self::Preparation(error) } else { - Self::InternalError(InternalValidationError::NonDeterministicPrepareError(error)) + Self::Internal(InternalValidationError::NonDeterministicPrepareError(error)) } } } diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 257377df3f48..a0c24fd44323 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -22,7 +22,7 @@ use crate::{ host::ResultSender, metrics::Metrics, worker_intf::{IdleWorker, WorkerHandle}, - InvalidCandidate, ValidationError, LOG_TARGET, + InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, }; use futures::{ channel::mpsc, @@ -342,27 +342,27 @@ fn handle_job_finish( }, Outcome::InvalidCandidate { err, idle_worker } => ( Some(idle_worker), - Err(ValidationError::InvalidCandidate(InvalidCandidate::WorkerReportedInvalid(err))), + Err(ValidationError::Invalid(InvalidCandidate::WorkerReportedInvalid(err))), None, ), - Outcome::InternalError { err } => (None, Err(ValidationError::InternalError(err)), None), + Outcome::InternalError { err } => (None, Err(ValidationError::Internal(err)), None), // Either the worker or the job timed out. Kill the worker in either case. Treated as // definitely-invalid, because if we timed out, there's no time left for a retry. Outcome::HardTimeout => - (None, Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)), None), + (None, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), None), // "Maybe invalid" errors (will retry). Outcome::WorkerIntfErr => ( None, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), None, ), Outcome::JobDied { err } => ( None, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousJobDeath(err))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))), None, ), Outcome::JobError { err } => - (None, Err(ValidationError::InvalidCandidate(InvalidCandidate::JobError(err))), None), + (None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))), None), }; queue.metrics.execute_finished(); diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index f67934e4171c..6049f51834e3 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -873,7 +873,7 @@ fn pulse_every(interval: std::time::Duration) -> impl futures::Stream #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::InvalidCandidate; + use crate::PossiblyInvalidError; use assert_matches::assert_matches; use futures::future::BoxFuture; use polkadot_node_core_pvf_common::{ @@ -1211,27 +1211,27 @@ pub(crate) mod tests { ); result_tx_pvf_1_1 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_pvf_1_1.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); result_tx_pvf_1_2 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_pvf_1_2.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); result_tx_pvf_2 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_pvf_2.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); } @@ -1337,7 +1337,7 @@ pub(crate) mod tests { assert_matches!(result_rx.now_or_never().unwrap().unwrap(), Err(PrepareError::TimedOut)); assert_matches!( result_rx_execute.now_or_never().unwrap().unwrap(), - Err(ValidationError::InternalError(_)) + Err(ValidationError::Internal(_)) ); // Reversed case: first send multiple precheck requests, then ask for an execution. @@ -1479,7 +1479,7 @@ pub(crate) mod tests { // The result should contain the error. let result = test.poll_and_recv_result(result_rx).await; - assert_matches!(result, Err(ValidationError::InternalError(_))); + assert_matches!(result, Err(ValidationError::Internal(_))); // Submit another execute request. We shouldn't try to prepare again, yet. let (result_tx_2, result_rx_2) = oneshot::channel(); @@ -1498,7 +1498,7 @@ pub(crate) mod tests { // The result should contain the original error. let result = test.poll_and_recv_result(result_rx_2).await; - assert_matches!(result, Err(ValidationError::InternalError(_))); + assert_matches!(result, Err(ValidationError::Internal(_))); // Pause for enough time to reset the cooldown for this failed prepare request. futures_timer::Delay::new(PREPARE_FAILURE_COOLDOWN).await; @@ -1538,11 +1538,11 @@ pub(crate) mod tests { // Send an error for the execution here, just so we can check the result receiver is still // alive. result_tx_3 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_3.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); } @@ -1581,10 +1581,7 @@ pub(crate) mod tests { // The result should contain the error. let result = test.poll_and_recv_result(result_rx).await; - assert_matches!( - result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::PrepareError(_))) - ); + assert_matches!(result, Err(ValidationError::Preparation(_))); // Submit another execute request. let (result_tx_2, result_rx_2) = oneshot::channel(); @@ -1603,10 +1600,7 @@ pub(crate) mod tests { // The result should contain the original error. let result = test.poll_and_recv_result(result_rx_2).await; - assert_matches!( - result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::PrepareError(_))) - ); + assert_matches!(result, Err(ValidationError::Preparation(_))); // Pause for enough time to reset the cooldown for this failed prepare request. futures_timer::Delay::new(PREPARE_FAILURE_COOLDOWN).await; @@ -1628,10 +1622,7 @@ pub(crate) mod tests { // The result should still contain the original error. let result = test.poll_and_recv_result(result_rx_3).await; - assert_matches!( - result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::PrepareError(_))) - ); + assert_matches!(result, Err(ValidationError::Preparation(_))); } // Test that multiple heads-up requests trigger preparation retries if the first one failed. diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index 7e7a13252548..3306a2461c15 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -103,7 +103,7 @@ mod worker_intf; #[cfg(feature = "test-utils")] pub mod testing; -pub use error::{InvalidCandidate, ValidationError}; +pub use error::{InvalidCandidate, PossiblyInvalidError, ValidationError}; pub use host::{start, Config, ValidationHost, EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME}; pub use metrics::Metrics; pub use priority::Priority; diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 5bdf49cc719e..4c81ac502dd4 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -162,7 +162,7 @@ async fn execute_job_terminates_on_timeout() { .await; match result { - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {}, + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) => {}, r => panic!("{:?}", r), } @@ -202,8 +202,8 @@ async fn ensure_parallel_execution() { assert_matches!( (res1, res2), ( - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)), - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) ) ); @@ -344,7 +344,7 @@ async fn deleting_prepared_artifact_does_not_dispute() { .await; match result { - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {}, + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) => {}, r => panic!("{:?}", r), } } diff --git a/polkadot/node/core/pvf/tests/it/process.rs b/polkadot/node/core/pvf/tests/it/process.rs index 725d060ab916..075d94373df0 100644 --- a/polkadot/node/core/pvf/tests/it/process.rs +++ b/polkadot/node/core/pvf/tests/it/process.rs @@ -19,7 +19,9 @@ use super::TestHost; use assert_matches::assert_matches; -use polkadot_node_core_pvf::{InvalidCandidate, PrepareError, ValidationError}; +use polkadot_node_core_pvf::{ + InvalidCandidate, PossiblyInvalidError, PrepareError, ValidationError, +}; use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams}; use procfs::process; use rusty_fork::rusty_fork_test; @@ -151,7 +153,7 @@ rusty_fork_test! { assert_matches!( result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) ); }) } @@ -217,7 +219,7 @@ rusty_fork_test! { assert_matches!( result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); }) } @@ -288,7 +290,7 @@ rusty_fork_test! { // Note that we get a more specific error if the job died than if the whole worker died. assert_matches!( result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousJobDeath(err))) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))) if err == "received signal: SIGKILL" ); }) From 40afc77c4e6efff56649174a35384499f601f8d4 Mon Sep 17 00:00:00 2001 From: Javier Viola Date: Tue, 21 Nov 2023 08:40:55 -0300 Subject: [PATCH 04/10] Add output positional arg to undying-collator cli (#2375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow up from https://github.com/paritytech/polkadot-sdk/pull/2370 to add `output` positional argument to undying-collator. cc @JoshOrndorff --------- Co-authored-by: Bastian Köcher --- .../undying/collator/src/cli.rs | 11 +++++++- .../undying/collator/src/main.rs | 26 ++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/polkadot/parachain/test-parachains/undying/collator/src/cli.rs b/polkadot/parachain/test-parachains/undying/collator/src/cli.rs index cd16133dbf19..d04122f2f689 100644 --- a/polkadot/parachain/test-parachains/undying/collator/src/cli.rs +++ b/polkadot/parachain/test-parachains/undying/collator/src/cli.rs @@ -18,6 +18,7 @@ use clap::Parser; use sc_cli::SubstrateCli; +use std::path::PathBuf; /// Sub-commands supported by the collator. #[derive(Debug, Parser)] @@ -34,6 +35,10 @@ pub enum Subcommand { /// Command for exporting the genesis state of the parachain #[derive(Debug, Parser)] pub struct ExportGenesisStateCommand { + /// Output file name or stdout if unspecified. + #[arg()] + pub output: Option, + /// Id of the parachain this collator collates for. #[arg(long, default_value_t = 100)] pub parachain_id: u32, @@ -50,7 +55,11 @@ pub struct ExportGenesisStateCommand { /// Command for exporting the genesis wasm file. #[derive(Debug, Parser)] -pub struct ExportGenesisWasmCommand {} +pub struct ExportGenesisWasmCommand { + /// Output file name or stdout if unspecified. + #[arg()] + pub output: Option, +} #[allow(missing_docs)] #[derive(Debug, Parser)] diff --git a/polkadot/parachain/test-parachains/undying/collator/src/main.rs b/polkadot/parachain/test-parachains/undying/collator/src/main.rs index e564e221f013..c6b338a20dc0 100644 --- a/polkadot/parachain/test-parachains/undying/collator/src/main.rs +++ b/polkadot/parachain/test-parachains/undying/collator/src/main.rs @@ -22,6 +22,10 @@ use polkadot_node_subsystem::messages::{CollationGenerationMessage, CollatorProt use polkadot_primitives::Id as ParaId; use sc_cli::{Error as SubstrateCliError, SubstrateCli}; use sp_core::hexdisplay::HexDisplay; +use std::{ + fs, + io::{self, Write}, +}; use test_parachain_undying_collator::Collator; mod cli; @@ -35,14 +39,30 @@ fn main() -> Result<()> { // `pov_size` and `pvf_complexity` need to match the ones that we start the collator // with. let collator = Collator::new(params.pov_size, params.pvf_complexity); - println!("0x{:?}", HexDisplay::from(&collator.genesis_head())); + + let output_buf = + format!("0x{:?}", HexDisplay::from(&collator.genesis_head())).into_bytes(); + + if let Some(output) = params.output { + std::fs::write(output, output_buf)?; + } else { + std::io::stdout().write_all(&output_buf)?; + } Ok::<_, Error>(()) }, - Some(cli::Subcommand::ExportGenesisWasm(_params)) => { + Some(cli::Subcommand::ExportGenesisWasm(params)) => { // We pass some dummy values for `pov_size` and `pvf_complexity` as these don't // matter for `wasm` export. - println!("0x{:?}", HexDisplay::from(&Collator::default().validation_code())); + let output_buf = + format!("0x{:?}", HexDisplay::from(&Collator::default().validation_code())) + .into_bytes(); + + if let Some(output) = params.output { + fs::write(output, output_buf)?; + } else { + io::stdout().write_all(&output_buf)?; + } Ok(()) }, From 552be4800d9e4b480f79a300fc531783e04be099 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 21 Nov 2023 12:52:46 +0100 Subject: [PATCH 05/10] PVF worker: switch on seccomp networking restrictions (#2221) --- polkadot/node/core/pvf/common/src/error.rs | 7 +- polkadot/node/core/pvf/common/src/execute.rs | 2 +- polkadot/node/core/pvf/common/src/lib.rs | 2 +- .../node/core/pvf/common/src/worker/mod.rs | 2 +- .../pvf/common/src/worker/security/seccomp.rs | 10 +- .../node/core/pvf/execute-worker/src/lib.rs | 26 ++-- .../node/core/pvf/prepare-worker/src/lib.rs | 24 ++-- .../node/core/pvf/src/execute/worker_intf.rs | 111 ++++++++++-------- polkadot/node/core/pvf/src/prepare/pool.rs | 4 +- .../node/core/pvf/src/prepare/worker_intf.rs | 45 ++++--- polkadot/node/core/pvf/src/security.rs | 75 ++++++++---- polkadot/node/core/pvf/tests/it/process.rs | 2 +- .../src/node/utility/pvf-host-and-workers.md | 38 +++--- 13 files changed, 202 insertions(+), 146 deletions(-) diff --git a/polkadot/node/core/pvf/common/src/error.rs b/polkadot/node/core/pvf/common/src/error.rs index 6bf05ece78ef..7db7f9a59451 100644 --- a/polkadot/node/core/pvf/common/src/error.rs +++ b/polkadot/node/core/pvf/common/src/error.rs @@ -77,7 +77,7 @@ pub enum PrepareError { #[codec(index = 9)] ClearWorkerDir(String), /// The preparation job process died, due to OOM, a seccomp violation, or some other factor. - JobDied(String), + JobDied { err: String, job_pid: i32 }, #[codec(index = 10)] /// Some error occurred when interfacing with the kernel. #[codec(index = 11)] @@ -96,7 +96,7 @@ impl PrepareError { match self { Prevalidation(_) | Preparation(_) | JobError(_) | OutOfMemory => true, IoErr(_) | - JobDied(_) | + JobDied { .. } | CreateTmpFile(_) | RenameTmpFile { .. } | ClearWorkerDir(_) | @@ -119,7 +119,8 @@ impl fmt::Display for PrepareError { JobError(err) => write!(f, "panic: {}", err), TimedOut => write!(f, "prepare: timeout"), IoErr(err) => write!(f, "prepare: io error while receiving response: {}", err), - JobDied(err) => write!(f, "prepare: prepare job died: {}", err), + JobDied { err, job_pid } => + write!(f, "prepare: prepare job with pid {job_pid} died: {err}"), CreateTmpFile(err) => write!(f, "prepare: error creating tmp file: {}", err), RenameTmpFile { err, src, dest } => write!(f, "prepare: error renaming tmp file ({:?} -> {:?}): {}", src, dest, err), diff --git a/polkadot/node/core/pvf/common/src/execute.rs b/polkadot/node/core/pvf/common/src/execute.rs index 89e7c8e471a5..aa1c1c539682 100644 --- a/polkadot/node/core/pvf/common/src/execute.rs +++ b/polkadot/node/core/pvf/common/src/execute.rs @@ -46,7 +46,7 @@ pub enum WorkerResponse { /// /// We cannot treat this as an internal error because malicious code may have killed the job. /// We still retry it, because in the non-malicious case it is likely spurious. - JobDied(String), + JobDied { err: String, job_pid: i32 }, /// An unexpected error occurred in the job process, e.g. failing to spawn a thread, panic, /// etc. /// diff --git a/polkadot/node/core/pvf/common/src/lib.rs b/polkadot/node/core/pvf/common/src/lib.rs index 282d2f7c41d0..278fa3fe821c 100644 --- a/polkadot/node/core/pvf/common/src/lib.rs +++ b/polkadot/node/core/pvf/common/src/lib.rs @@ -53,7 +53,7 @@ pub struct SecurityStatus { pub can_enable_landlock: bool, /// Whether the seccomp features we use are fully available on this system. pub can_enable_seccomp: bool, - // Whether we are able to unshare the user namespace and change the filesystem root. + /// Whether we are able to unshare the user namespace and change the filesystem root. pub can_unshare_user_namespace_and_change_root: bool, } diff --git a/polkadot/node/core/pvf/common/src/worker/mod.rs b/polkadot/node/core/pvf/common/src/worker/mod.rs index 86f47acccac6..d4f9bbc27ea6 100644 --- a/polkadot/node/core/pvf/common/src/worker/mod.rs +++ b/polkadot/node/core/pvf/common/src/worker/mod.rs @@ -219,7 +219,7 @@ pub fn run_worker( #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] mut worker_dir_path: PathBuf, node_version: Option<&str>, worker_version: Option<&str>, - #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] security_status: &SecurityStatus, + security_status: &SecurityStatus, mut event_loop: F, ) where F: FnMut(UnixStream, PathBuf) -> io::Result, diff --git a/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs b/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs index 5539ad284400..c3822d3c4c69 100644 --- a/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs +++ b/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs @@ -67,11 +67,9 @@ //! //! # Action on syscall violations //! -//! On syscall violations we currently only log, to make sure this works correctly before enforcing. -//! -//! In the future, when a forbidden syscall is attempted we immediately kill the process in order to -//! prevent the attacker from doing anything else. In execution, this will result in voting against -//! the candidate. +//! When a forbidden syscall is attempted we immediately kill the process in order to prevent the +//! attacker from doing anything else. In execution, this will result in voting against the +//! candidate. use crate::{ worker::{stringify_panic_payload, WorkerKind}, @@ -82,7 +80,7 @@ use std::{collections::BTreeMap, path::Path}; /// The action to take on caught syscalls. #[cfg(not(test))] -const CAUGHT_ACTION: SeccompAction = SeccompAction::Log; +const CAUGHT_ACTION: SeccompAction = SeccompAction::KillProcess; /// Don't kill the process when testing. #[cfg(test)] const CAUGHT_ACTION: SeccompAction = SeccompAction::Errno(libc::EACCES as u32); diff --git a/polkadot/node/core/pvf/execute-worker/src/lib.rs b/polkadot/node/core/pvf/execute-worker/src/lib.rs index 9ec811686b89..d82a8fca65d3 100644 --- a/polkadot/node/core/pvf/execute-worker/src/lib.rs +++ b/polkadot/node/core/pvf/execute-worker/src/lib.rs @@ -222,12 +222,6 @@ pub fn worker_entrypoint( }, }; - gum::trace!( - target: LOG_TARGET, - %worker_pid, - "worker: sending response to host: {:?}", - response - ); send_response(&mut stream, response)?; } }, @@ -360,7 +354,7 @@ fn handle_child_process( /// - The response, either `Ok` or some error state. fn handle_parent_process( mut pipe_read: PipeReader, - child: Pid, + job_pid: Pid, worker_pid: u32, usage_before: Usage, timeout: Duration, @@ -373,10 +367,11 @@ fn handle_parent_process( // Should retry at any rate. .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string()))?; - let status = nix::sys::wait::waitpid(child, None); + let status = nix::sys::wait::waitpid(job_pid, None); gum::trace!( target: LOG_TARGET, %worker_pid, + %job_pid, "execute worker received wait status from job: {:?}", status, ); @@ -396,6 +391,7 @@ fn handle_parent_process( gum::warn!( target: LOG_TARGET, %worker_pid, + %job_pid, "execute job took {}ms cpu time, exceeded execute timeout {}ms", cpu_tv.as_millis(), timeout.as_millis(), @@ -428,6 +424,7 @@ fn handle_parent_process( gum::warn!( target: LOG_TARGET, %worker_pid, + %job_pid, "execute job error: {}", job_error, ); @@ -443,15 +440,18 @@ fn handle_parent_process( // // The job gets SIGSYS on seccomp violations, but this signal may have been sent for some // other reason, so we still need to check for seccomp violations elsewhere. - Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => - Ok(WorkerResponse::JobDied(format!("received signal: {signal:?}"))), + Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => Ok(WorkerResponse::JobDied { + err: format!("received signal: {signal:?}"), + job_pid: job_pid.as_raw(), + }), Err(errno) => Ok(internal_error_from_errno("waitpid", errno)), // It is within an attacker's power to send an unexpected exit status. So we cannot treat // this as an internal error (which would make us abstain), but must vote against. - Ok(unexpected_wait_status) => Ok(WorkerResponse::JobDied(format!( - "unexpected status from wait: {unexpected_wait_status:?}" - ))), + Ok(unexpected_wait_status) => Ok(WorkerResponse::JobDied { + err: format!("unexpected status from wait: {unexpected_wait_status:?}"), + job_pid: job_pid.as_raw(), + }), } } diff --git a/polkadot/node/core/pvf/prepare-worker/src/lib.rs b/polkadot/node/core/pvf/prepare-worker/src/lib.rs index 34e6a78c26ae..499e87c6e207 100644 --- a/polkadot/node/core/pvf/prepare-worker/src/lib.rs +++ b/polkadot/node/core/pvf/prepare-worker/src/lib.rs @@ -257,9 +257,9 @@ pub fn worker_entrypoint( handle_parent_process( pipe_reader, + worker_pid, child, temp_artifact_dest.clone(), - worker_pid, usage_before, preparation_timeout, ) @@ -506,9 +506,9 @@ fn handle_child_process( /// - If the child process timeout, it returns `PrepareError::TimedOut`. fn handle_parent_process( mut pipe_read: PipeReader, - child: Pid, - temp_artifact_dest: PathBuf, worker_pid: u32, + job_pid: Pid, + temp_artifact_dest: PathBuf, usage_before: Usage, timeout: Duration, ) -> Result { @@ -518,10 +518,11 @@ fn handle_parent_process( .read_to_end(&mut received_data) .map_err(|err| PrepareError::IoErr(err.to_string()))?; - let status = nix::sys::wait::waitpid(child, None); + let status = nix::sys::wait::waitpid(job_pid, None); gum::trace!( target: LOG_TARGET, %worker_pid, + %job_pid, "prepare worker received wait status from job: {:?}", status, ); @@ -539,6 +540,7 @@ fn handle_parent_process( gum::warn!( target: LOG_TARGET, %worker_pid, + %job_pid, "prepare job took {}ms cpu time, exceeded prepare timeout {}ms", cpu_tv.as_millis(), timeout.as_millis(), @@ -573,6 +575,7 @@ fn handle_parent_process( gum::debug!( target: LOG_TARGET, %worker_pid, + %job_pid, "worker: writing artifact to {}", temp_artifact_dest.display(), ); @@ -593,15 +596,18 @@ fn handle_parent_process( // // The job gets SIGSYS on seccomp violations, but this signal may have been sent for some // other reason, so we still need to check for seccomp violations elsewhere. - Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => - Err(PrepareError::JobDied(format!("received signal: {signal:?}"))), + Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => Err(PrepareError::JobDied { + err: format!("received signal: {signal:?}"), + job_pid: job_pid.as_raw(), + }), Err(errno) => Err(error_from_errno("waitpid", errno)), // An attacker can make the child process return any exit status it wants. So we can treat // all unexpected cases the same way. - Ok(unexpected_wait_status) => Err(PrepareError::JobDied(format!( - "unexpected status from wait: {unexpected_wait_status:?}" - ))), + Ok(unexpected_wait_status) => Err(PrepareError::JobDied { + err: format!("unexpected status from wait: {unexpected_wait_status:?}"), + job_pid: job_pid.as_raw(), + }), } } diff --git a/polkadot/node/core/pvf/src/execute/worker_intf.rs b/polkadot/node/core/pvf/src/execute/worker_intf.rs index bf44ba017250..4faaf13e62f7 100644 --- a/polkadot/node/core/pvf/src/execute/worker_intf.rs +++ b/polkadot/node/core/pvf/src/execute/worker_intf.rs @@ -33,7 +33,7 @@ use polkadot_node_core_pvf_common::{ execute::{Handshake, WorkerResponse}, worker_dir, SecurityStatus, }; -use polkadot_parachain_primitives::primitives::ValidationResult; +use polkadot_parachain_primitives::primitives::{ValidationCodeHash, ValidationResult}; use polkadot_primitives::ExecutorParams; use std::{path::Path, time::Duration}; use tokio::{io, net::UnixStream}; @@ -156,6 +156,16 @@ pub async fn start_work( let response = futures::select! { response = recv_response(&mut stream).fuse() => { match response { + Ok(response) => + handle_response( + response, + pid, + &artifact.id.code_hash, + &artifact_path, + execution_timeout, + audit_log_file + ) + .await, Err(error) => { gum::warn!( target: LOG_TARGET, @@ -164,56 +174,9 @@ pub async fn start_work( ?error, "failed to recv an execute response", ); - // The worker died. Check if it was due to a seccomp violation. - // - // NOTE: Log, but don't change the outcome. Not all validators may have - // auditing enabled, so we don't want attackers to abuse a non-deterministic - // outcome. - for syscall in security::check_seccomp_violations_for_worker(audit_log_file, pid).await { - gum::error!( - target: LOG_TARGET, - worker_pid = %pid, - %syscall, - validation_code_hash = ?artifact.id.code_hash, - ?artifact_path, - "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" - ); - } return Outcome::WorkerIntfErr }, - Ok(response) => { - // Check if any syscall violations occurred during the job. For now this is - // only informative, as we are not enforcing the seccomp policy yet. - for syscall in security::check_seccomp_violations_for_worker(audit_log_file, pid).await { - gum::error!( - target: LOG_TARGET, - worker_pid = %pid, - %syscall, - validation_code_hash = ?artifact.id.code_hash, - ?artifact_path, - "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" - ); - } - - if let WorkerResponse::Ok{duration, ..} = response { - if duration > execution_timeout { - // The job didn't complete within the timeout. - gum::warn!( - target: LOG_TARGET, - worker_pid = %pid, - "execute job took {}ms cpu time, exceeded execution timeout {}ms.", - duration.as_millis(), - execution_timeout.as_millis(), - ); - - // Return a timeout error. - return Outcome::HardTimeout - } - } - - response - }, } }, _ = Delay::new(timeout).fuse() => { @@ -238,7 +201,7 @@ pub async fn start_work( idle_worker: IdleWorker { stream, pid, worker_dir }, }, WorkerResponse::JobTimedOut => Outcome::HardTimeout, - WorkerResponse::JobDied(err) => Outcome::JobDied { err }, + WorkerResponse::JobDied { err, job_pid: _ } => Outcome::JobDied { err }, WorkerResponse::JobError(err) => Outcome::JobError { err }, WorkerResponse::InternalError(err) => Outcome::InternalError { err }, @@ -247,6 +210,56 @@ pub async fn start_work( .await } +/// Handles the case where we successfully received response bytes on the host from the child. +/// +/// Here we know the artifact exists, but is still located in a temporary file which will be cleared +/// by [`with_worker_dir_setup`]. +async fn handle_response( + response: WorkerResponse, + worker_pid: u32, + validation_code_hash: &ValidationCodeHash, + artifact_path: &Path, + execution_timeout: Duration, + audit_log_file: Option, +) -> WorkerResponse { + if let WorkerResponse::Ok { duration, .. } = response { + if duration > execution_timeout { + // The job didn't complete within the timeout. + gum::warn!( + target: LOG_TARGET, + worker_pid, + "execute job took {}ms cpu time, exceeded execution timeout {}ms.", + duration.as_millis(), + execution_timeout.as_millis(), + ); + + // Return a timeout error. + return WorkerResponse::JobTimedOut + } + } + + if let WorkerResponse::JobDied { err: _, job_pid } = response { + // The job died. Check if it was due to a seccomp violation. + // + // NOTE: Log, but don't change the outcome. Not all validators may have + // auditing enabled, so we don't want attackers to abuse a non-deterministic + // outcome. + for syscall in security::check_seccomp_violations_for_job(audit_log_file, job_pid).await { + gum::error!( + target: LOG_TARGET, + %worker_pid, + %job_pid, + %syscall, + ?validation_code_hash, + ?artifact_path, + "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" + ); + } + } + + response +} + /// Create a temporary file for an artifact in the worker cache, execute the given future/closure /// passing the file path in, and clean up the worker cache. /// diff --git a/polkadot/node/core/pvf/src/prepare/pool.rs b/polkadot/node/core/pvf/src/prepare/pool.rs index 21af21e5b028..4901be9fe1b7 100644 --- a/polkadot/node/core/pvf/src/prepare/pool.rs +++ b/polkadot/node/core/pvf/src/prepare/pool.rs @@ -388,14 +388,14 @@ fn handle_mux( Ok(()) }, // The worker might still be usable, but we kill it just in case. - Outcome::JobDied(err) => { + Outcome::JobDied { err, job_pid } => { if attempt_retire(metrics, spawned, worker) { reply( from_pool, FromPool::Concluded { worker, rip: true, - result: Err(PrepareError::JobDied(err)), + result: Err(PrepareError::JobDied { err, job_pid }), }, )?; } diff --git a/polkadot/node/core/pvf/src/prepare/worker_intf.rs b/polkadot/node/core/pvf/src/prepare/worker_intf.rs index e7f142a46bb8..318aea7295de 100644 --- a/polkadot/node/core/pvf/src/prepare/worker_intf.rs +++ b/polkadot/node/core/pvf/src/prepare/worker_intf.rs @@ -104,7 +104,7 @@ pub enum Outcome { /// The preparation job process died, due to OOM, a seccomp violation, or some other factor. /// /// The worker might still be usable, but we kill it just in case. - JobDied(String), + JobDied { err: String, job_pid: i32 }, } /// Given the idle token of a worker and parameters of work, communicates with the worker and @@ -160,19 +160,7 @@ pub async fn start_work( match result { // Received bytes from worker within the time limit. - Ok(Ok(prepare_worker_result)) => { - // Check if any syscall violations occurred during the job. For now this is only - // informative, as we are not enforcing the seccomp policy yet. - for syscall in security::check_seccomp_violations_for_worker(audit_log_file, pid).await { - gum::error!( - target: LOG_TARGET, - worker_pid = %pid, - %syscall, - ?pvf, - "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" - ); - } - + Ok(Ok(prepare_worker_result)) => handle_response( metrics, IdleWorker { stream, pid, worker_dir }, @@ -182,9 +170,9 @@ pub async fn start_work( &pvf, &cache_path, preparation_timeout, + audit_log_file, ) - .await - }, + .await, Ok(Err(err)) => { // Communication error within the time limit. gum::warn!( @@ -221,15 +209,36 @@ async fn handle_response( worker_pid: u32, tmp_file: PathBuf, pvf: &PvfPrepData, - cache_path: &PathBuf, + cache_path: &Path, preparation_timeout: Duration, + audit_log_file: Option, ) -> Outcome { let PrepareWorkerSuccess { checksum, stats: PrepareStats { cpu_time_elapsed, memory_stats } } = match result.clone() { Ok(result) => result, // Timed out on the child. This should already be logged by the child. Err(PrepareError::TimedOut) => return Outcome::TimedOut, - Err(PrepareError::JobDied(err)) => return Outcome::JobDied(err), + Err(PrepareError::JobDied { err, job_pid }) => { + // The job died. Check if it was due to a seccomp violation. + // + // NOTE: Log, but don't change the outcome. Not all validators may have + // auditing enabled, so we don't want attackers to abuse a non-deterministic + // outcome. + for syscall in + security::check_seccomp_violations_for_job(audit_log_file, job_pid).await + { + gum::error!( + target: LOG_TARGET, + %worker_pid, + %job_pid, + %syscall, + ?pvf, + "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" + ); + } + + return Outcome::JobDied { err, job_pid } + }, Err(PrepareError::OutOfMemory) => return Outcome::OutOfMemory, Err(err) => return Outcome::Concluded { worker, result: Err(err) }, }; diff --git a/polkadot/node/core/pvf/src/security.rs b/polkadot/node/core/pvf/src/security.rs index 0c0c5f401663..8c06c68392f4 100644 --- a/polkadot/node/core/pvf/src/security.rs +++ b/polkadot/node/core/pvf/src/security.rs @@ -178,9 +178,15 @@ async fn check_can_unshare_user_namespace_and_change_root( let stderr = std::str::from_utf8(&output.stderr) .expect("child process writes a UTF-8 string to stderr; qed") .trim(); - Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( - format!("not available: {}", stderr) - )) + if stderr.is_empty() { + Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( + "not available".into() + )) + } else { + Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( + format!("not available: {}", stderr) + )) + } }, Err(err) => Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( @@ -208,16 +214,25 @@ async fn check_landlock( if #[cfg(target_os = "linux")] { match tokio::process::Command::new(prepare_worker_program_path) .arg("--check-can-enable-landlock") - .status() + .output() .await { - Ok(status) if status.success() => Ok(()), - Ok(_status) => { + Ok(output) if output.status.success() => Ok(()), + Ok(output) => { let abi = polkadot_node_core_pvf_common::worker::security::landlock::LANDLOCK_ABI as u8; - Err(SecureModeError::CannotEnableLandlock( - format!("landlock ABI {} not available", abi) - )) + let stderr = std::str::from_utf8(&output.stderr) + .expect("child process writes a UTF-8 string to stderr; qed") + .trim(); + if stderr.is_empty() { + Err(SecureModeError::CannotEnableLandlock( + format!("landlock ABI {} not available", abi) + )) + } else { + Err(SecureModeError::CannotEnableLandlock( + format!("not available: {}", stderr) + )) + } }, Err(err) => Err(SecureModeError::CannotEnableLandlock( @@ -238,7 +253,7 @@ async fn check_landlock( /// to running the check in a worker, we try it... in a worker. The expected return status is 0 on /// success and -1 on failure. async fn check_seccomp( - #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] + #[cfg_attr(not(all(target_os = "linux", target_arch = "x86_64")), allow(unused_variables))] prepare_worker_program_path: &Path, ) -> SecureModeResult { cfg_if::cfg_if! { @@ -247,14 +262,24 @@ async fn check_seccomp( if #[cfg(target_arch = "x86_64")] { match tokio::process::Command::new(prepare_worker_program_path) .arg("--check-can-enable-seccomp") - .status() + .output() .await { - Ok(status) if status.success() => Ok(()), - Ok(_status) => - Err(SecureModeError::CannotEnableSeccomp( - "not available".into() - )), + Ok(output) if output.status.success() => Ok(()), + Ok(output) => { + let stderr = std::str::from_utf8(&output.stderr) + .expect("child process writes a UTF-8 string to stderr; qed") + .trim(); + if stderr.is_empty() { + Err(SecureModeError::CannotEnableSeccomp( + "not available".into() + )) + } else { + Err(SecureModeError::CannotEnableSeccomp( + format!("not available: {}", stderr) + )) + } + }, Err(err) => Err(SecureModeError::CannotEnableSeccomp( format!("could not start child process: {}", err) @@ -320,25 +345,25 @@ impl AuditLogFile { } } -/// Check if a seccomp violation occurred for the given worker. As the syslog may be in a different -/// location, or seccomp auditing may be disabled, this function provides a best-effort attempt -/// only. +/// Check if a seccomp violation occurred for the given job process. As the syslog may be in a +/// different location, or seccomp auditing may be disabled, this function provides a best-effort +/// attempt only. /// /// The `audit_log_file` must have been obtained before the job started. It only allows reading /// entries that were written since it was obtained, so that we do not consider events from previous /// processes with the same pid. This can still be racy, but it's unlikely and fine for a /// best-effort attempt. -pub async fn check_seccomp_violations_for_worker( +pub async fn check_seccomp_violations_for_job( audit_log_file: Option, - worker_pid: u32, + job_pid: i32, ) -> Vec { - let audit_event_pid_field = format!("pid={worker_pid}"); + let audit_event_pid_field = format!("pid={job_pid}"); let audit_log_file = match audit_log_file { Some(file) => { - gum::debug!( + gum::trace!( target: LOG_TARGET, - %worker_pid, + %job_pid, audit_log_path = ?file.path, "checking audit log for seccomp violations", ); @@ -347,7 +372,7 @@ pub async fn check_seccomp_violations_for_worker( None => { gum::warn!( target: LOG_TARGET, - %worker_pid, + %job_pid, "could not open either {AUDIT_LOG_PATH} or {SYSLOG_PATH} for reading audit logs" ); return vec![] diff --git a/polkadot/node/core/pvf/tests/it/process.rs b/polkadot/node/core/pvf/tests/it/process.rs index 075d94373df0..b742acb15d02 100644 --- a/polkadot/node/core/pvf/tests/it/process.rs +++ b/polkadot/node/core/pvf/tests/it/process.rs @@ -248,7 +248,7 @@ rusty_fork_test! { // Note that we get a more specific error if the job died than if the whole worker died. assert_matches!( result, - Err(PrepareError::JobDied(err)) if err == "received signal: SIGKILL" + Err(PrepareError::JobDied{ err, job_pid: _ }) if err == "received signal: SIGKILL" ); }) } diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md index 74d88ba3ad99..56bdd48bc0c3 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md @@ -38,16 +38,22 @@ getting backed and honest backers getting slashed. We currently know of the following specific cases that will lead to a retried execution request: -1. **OOM:** The host might have been temporarily low on memory due to other - processes running on the same machine. **NOTE:** This case will lead to - voting against the candidate (and possibly a dispute) if the retry is still - not successful. -2. **Artifact missing:** The prepared artifact might have been deleted due to +1. **OOM:** We have memory limits to try to prevent attackers from exhausting + host memory. If the memory limit is hit, we kill the job process and retry + the job. Alternatively, the host might have been temporarily low on memory + due to other processes running on the same machine. **NOTE:** This case will + lead to voting against the candidate (and possibly a dispute) if the retry is + still not successful. +2. **Syscall violations:** If the job attempts a system call that is blocked by + the sandbox's security policy, the job process is immediately killed and we + retry. **NOTE:** In the future, if we have a proper way to detect that the + job died due to a security violation, it might make sense not to retry in + this case. +3. **Artifact missing:** The prepared artifact might have been deleted due to operator error or some bug in the system. -3. **Job errors:** For example, the worker thread panicked for some - indeterminate reason, which may or may not be independent of the candidate or - PVF. -4. **Internal errors:** See "Internal Errors" section. In this case, after the +4. **Job errors:** For example, the job process panicked for some indeterminate + reason, which may or may not be independent of the candidate or PVF. +5. **Internal errors:** See "Internal Errors" section. In this case, after the retry we abstain from voting. ### Preparation timeouts @@ -159,16 +165,14 @@ data on the host machine. *Currently this is only supported on Linux.* - +### Restricting networking - +We also disable networking on PVF threads by disabling certain syscalls, such as +the creation of sockets. This prevents attackers from either downloading +payloads or communicating sensitive data from the validator's machine to the +outside world. - - - - - - +*Currently this is only supported on Linux.* ### Clearing env vars From b25d29a50247f284cde5b24e270b6db7b4a81e7e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:22:42 +0100 Subject: [PATCH 06/10] cumulus-consensus-common: block import: `delayed_best_block` flag added (#2001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the `delayed_best_block` flag to `ParachainBlockImport`. If not set, the `params.fork_choice` is not updated (to `ForkChoiceStrategy::Custom`) during the block import. When `delayed_best_block` is set to `false` all parachain blocks on the [longest fork](https://github.com/paritytech/polkadot-sdk/blob/552be4800d9e4b480f79a300fc531783e04be099/substrate/client/service/src/client/client.rs#L708-L709) will be notified as the best block, allowing transaction pool to be updated with every imported block. Otherwise imported blocks will not be notified as best blocks (`fork_choice=ForkChoiceStrategy::Custom(false)`), and transaction pool would be updated only with best block received from relay-chain. Improvement for: #1202 --------- Co-authored-by: Bastian Köcher --- cumulus/client/consensus/common/src/lib.rs | 39 +++++++++++++++----- cumulus/client/consensus/common/src/tests.rs | 6 ++- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/cumulus/client/consensus/common/src/lib.rs b/cumulus/client/consensus/common/src/lib.rs index 08bceabb2bd4..cebe34e7ea58 100644 --- a/cumulus/client/consensus/common/src/lib.rs +++ b/cumulus/client/consensus/common/src/lib.rs @@ -111,12 +111,15 @@ impl ParachainConsensus for Box + Send + /// Parachain specific block import. /// -/// This is used to set `block_import_params.fork_choice` to `false` as long as the block origin is -/// not `NetworkInitialSync`. The best block for parachains is determined by the relay chain. -/// Meaning we will update the best block, as it is included by the relay-chain. +/// Specialized block import for parachains. It supports to delay setting the best block until the +/// relay chain has included a candidate in its best block. By default the delayed best block +/// setting is disabled. The block import also monitors the imported blocks and prunes by default if +/// there are too many blocks at the same height. Too many blocks at the same height can for example +/// happen if the relay chain is rejecting the parachain blocks in the validation. pub struct ParachainBlockImport { inner: BI, monitor: Option>>, + delayed_best_block: bool, } impl> ParachainBlockImport { @@ -141,13 +144,27 @@ impl> ParachainBlockImport let monitor = level_limit.map(|level_limit| SharedData::new(LevelMonitor::new(level_limit, backend))); - Self { inner, monitor } + Self { inner, monitor, delayed_best_block: false } + } + + /// Create a new instance which delays setting the best block. + /// + /// The number of leaves per level limit is set to `LevelLimit::Default`. + pub fn new_with_delayed_best_block(inner: BI, backend: Arc) -> Self { + Self { + delayed_best_block: true, + ..Self::new_with_limit(inner, backend, LevelLimit::Default) + } } } impl Clone for ParachainBlockImport { fn clone(&self) -> Self { - ParachainBlockImport { inner: self.inner.clone(), monitor: self.monitor.clone() } + ParachainBlockImport { + inner: self.inner.clone(), + monitor: self.monitor.clone(), + delayed_best_block: self.delayed_best_block, + } } } @@ -182,11 +199,13 @@ where params.finalized = true; } - // Best block is determined by the relay chain, or if we are doing the initial sync - // we import all blocks as new best. - params.fork_choice = Some(sc_consensus::ForkChoiceStrategy::Custom( - params.origin == sp_consensus::BlockOrigin::NetworkInitialSync, - )); + if self.delayed_best_block { + // Best block is determined by the relay chain, or if we are doing the initial sync + // we import all blocks as new best. + params.fork_choice = Some(sc_consensus::ForkChoiceStrategy::Custom( + params.origin == sp_consensus::BlockOrigin::NetworkInitialSync, + )); + } let maybe_lock = self.monitor.as_ref().map(|monitor_lock| { let mut monitor = monitor_lock.shared_data_locked(); diff --git a/cumulus/client/consensus/common/src/tests.rs b/cumulus/client/consensus/common/src/tests.rs index 9658a0add790..597d1ab2acc2 100644 --- a/cumulus/client/consensus/common/src/tests.rs +++ b/cumulus/client/consensus/common/src/tests.rs @@ -1124,7 +1124,8 @@ fn find_potential_parents_aligned_with_pending() { let backend = Arc::new(Backend::new_test(1000, 1)); let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); - let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + let mut para_import = + ParachainBlockImport::new_with_delayed_best_block(client.clone(), backend.clone()); let relay_parent = relay_hash_from_block_num(10); // Choose different relay parent for alternative chain to get new hashes. @@ -1279,7 +1280,8 @@ fn find_potential_parents_aligned_no_pending() { let backend = Arc::new(Backend::new_test(1000, 1)); let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); - let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + let mut para_import = + ParachainBlockImport::new_with_delayed_best_block(client.clone(), backend.clone()); let relay_parent = relay_hash_from_block_num(10); // Choose different relay parent for alternative chain to get new hashes. From b3841b6b71cd3707d80e90492059d1a2e3bd33a6 Mon Sep 17 00:00:00 2001 From: Francisco Aguirre Date: Tue, 21 Nov 2023 16:09:40 +0100 Subject: [PATCH 07/10] Different XCM builders, default one requires fee payment (#2253) Adding on top of the new builder pattern for creating XCM programs, I'm adding some more APIs: ```rust let paying_fees: Xcm<()> = Xcm::builder() // Only allow paying for fees .withdraw_asset() // First instruction has to load the holding register .buy_execution() // Second instruction has to be `buy_execution` .build(); let paying_fees_invalid: Xcm<()> = Xcm::builder() .withdraw_asset() .build(); // Invalid, need to pay for fees let not_paying_fees: Xcm<()> = Xcm::builder_unpaid() .unpaid_execution() // Needed .withdraw_asset() .deposit_asset() .build(); let all_goes: Xcm<()> = Xcm::builder_unsafe() // You can do anything .withdraw_asset() .deposit_asset() .build(); ``` The invalid bits are because the methods don't even exist on the types that you'd want to call them on. --------- Co-authored-by: command-bot <> --- Cargo.lock | 1 + polkadot/xcm/procedural/Cargo.toml | 1 + .../xcm/procedural/src/builder_pattern.rs | 287 ++++++++++++++++-- polkadot/xcm/procedural/src/lib.rs | 6 +- .../xcm/procedural/tests/builder_pattern.rs | 81 +++++ polkadot/xcm/procedural/tests/ui.rs | 2 +- .../badly_formatted_attribute.rs | 32 ++ .../badly_formatted_attribute.stderr | 5 + .../buy_execution_named_fields.rs | 30 ++ .../buy_execution_named_fields.stderr | 5 + .../loads_holding_no_operands.rs | 32 ++ .../loads_holding_no_operands.stderr | 6 + .../ui/builder_pattern/no_buy_execution.rs | 29 ++ .../builder_pattern/no_buy_execution.stderr | 6 + .../ui/builder_pattern/no_unpaid_execution.rs | 29 ++ .../no_unpaid_execution.stderr | 6 + .../builder_pattern/unexpected_attribute.rs | 32 ++ .../unexpected_attribute.stderr | 5 + .../unpaid_execution_named_fields.rs | 30 ++ .../unpaid_execution_named_fields.stderr | 5 + .../wrong_target.rs} | 0 .../wrong_target.stderr} | 2 +- polkadot/xcm/src/v3/mod.rs | 4 + polkadot/xcm/xcm-simulator/example/src/lib.rs | 19 -- prdoc/pr_2253.prdoc | 24 ++ 25 files changed, 625 insertions(+), 54 deletions(-) create mode 100644 polkadot/xcm/procedural/tests/builder_pattern.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr rename polkadot/xcm/procedural/tests/ui/{builder_pattern.rs => builder_pattern/wrong_target.rs} (100%) rename polkadot/xcm/procedural/tests/ui/{builder_pattern.stderr => builder_pattern/wrong_target.stderr} (63%) create mode 100644 prdoc/pr_2253.prdoc diff --git a/Cargo.lock b/Cargo.lock index 697f232f9719..88cefe7bbef5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21512,6 +21512,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", + "staging-xcm", "syn 2.0.38", "trybuild", ] diff --git a/polkadot/xcm/procedural/Cargo.toml b/polkadot/xcm/procedural/Cargo.toml index 33c2a94be0e4..8ab27c91dae1 100644 --- a/polkadot/xcm/procedural/Cargo.toml +++ b/polkadot/xcm/procedural/Cargo.toml @@ -18,3 +18,4 @@ Inflector = "0.11.4" [dev-dependencies] trybuild = { version = "1.0.74", features = ["diff"] } +xcm = { package = "staging-xcm", path = ".." } diff --git a/polkadot/xcm/procedural/src/builder_pattern.rs b/polkadot/xcm/procedural/src/builder_pattern.rs index ebad54e972b6..1cb795ea9b20 100644 --- a/polkadot/xcm/procedural/src/builder_pattern.rs +++ b/polkadot/xcm/procedural/src/builder_pattern.rs @@ -17,56 +17,83 @@ //! Derive macro for creating XCMs with a builder pattern use inflector::Inflector; -use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::{format_ident, quote}; use syn::{ - parse_macro_input, Data, DeriveInput, Error, Expr, ExprLit, Fields, Lit, Meta, MetaNameValue, + Data, DataEnum, DeriveInput, Error, Expr, ExprLit, Fields, Ident, Lit, Meta, MetaNameValue, + Result, Variant, }; -pub fn derive(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input as DeriveInput); - let builder_impl = match &input.data { - Data::Enum(data_enum) => generate_methods_for_enum(input.ident, data_enum), - _ => - return Error::new_spanned(&input, "Expected the `Instruction` enum") - .to_compile_error() - .into(), +pub fn derive(input: DeriveInput) -> Result { + let data_enum = match &input.data { + Data::Enum(data_enum) => data_enum, + _ => return Err(Error::new_spanned(&input, "Expected the `Instruction` enum")), }; + let builder_raw_impl = generate_builder_raw_impl(&input.ident, data_enum); + let builder_impl = generate_builder_impl(&input.ident, data_enum)?; + let builder_unpaid_impl = generate_builder_unpaid_impl(&input.ident, data_enum)?; let output = quote! { - pub struct XcmBuilder(Vec>); + /// A trait for types that track state inside the XcmBuilder + pub trait XcmBuilderState {} + + /// Access to all the instructions + pub enum AnythingGoes {} + /// You need to pay for execution + pub enum PaymentRequired {} + /// The holding register was loaded, now to buy execution + pub enum LoadedHolding {} + /// Need to explicitly state it won't pay for fees + pub enum ExplicitUnpaidRequired {} + + impl XcmBuilderState for AnythingGoes {} + impl XcmBuilderState for PaymentRequired {} + impl XcmBuilderState for LoadedHolding {} + impl XcmBuilderState for ExplicitUnpaidRequired {} + + /// Type used to build XCM programs + pub struct XcmBuilder { + pub(crate) instructions: Vec>, + pub state: core::marker::PhantomData, + } + impl Xcm { - pub fn builder() -> XcmBuilder { - XcmBuilder::(Vec::new()) + pub fn builder() -> XcmBuilder { + XcmBuilder:: { + instructions: Vec::new(), + state: core::marker::PhantomData, + } + } + pub fn builder_unpaid() -> XcmBuilder { + XcmBuilder:: { + instructions: Vec::new(), + state: core::marker::PhantomData, + } + } + pub fn builder_unsafe() -> XcmBuilder { + XcmBuilder:: { + instructions: Vec::new(), + state: core::marker::PhantomData, + } } } #builder_impl + #builder_unpaid_impl + #builder_raw_impl }; - output.into() + Ok(output) } -fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> TokenStream2 { +fn generate_builder_raw_impl(name: &Ident, data_enum: &DataEnum) -> TokenStream2 { let methods = data_enum.variants.iter().map(|variant| { let variant_name = &variant.ident; let method_name_string = &variant_name.to_string().to_snake_case(); let method_name = syn::Ident::new(&method_name_string, variant_name.span()); - let docs: Vec<_> = variant - .attrs - .iter() - .filter_map(|attr| match &attr.meta { - Meta::NameValue(MetaNameValue { - value: Expr::Lit(ExprLit { lit: Lit::Str(literal), .. }), - .. - }) if attr.path().is_ident("doc") => Some(literal.value()), - _ => None, - }) - .map(|doc| syn::parse_str::(&format!("/// {}", doc)).unwrap()) - .collect(); + let docs = get_doc_comments(&variant); let method = match &variant.fields { Fields::Unit => { quote! { pub fn #method_name(mut self) -> Self { - self.0.push(#name::::#variant_name); + self.instructions.push(#name::::#variant_name); self } } @@ -81,7 +108,7 @@ fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> Tok let arg_types: Vec<_> = fields.unnamed.iter().map(|field| &field.ty).collect(); quote! { pub fn #method_name(mut self, #(#arg_names: #arg_types),*) -> Self { - self.0.push(#name::::#variant_name(#(#arg_names),*)); + self.instructions.push(#name::::#variant_name(#(#arg_names),*)); self } } @@ -91,7 +118,7 @@ fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> Tok let arg_types: Vec<_> = fields.named.iter().map(|field| &field.ty).collect(); quote! { pub fn #method_name(mut self, #(#arg_names: #arg_types),*) -> Self { - self.0.push(#name::::#variant_name { #(#arg_names),* }); + self.instructions.push(#name::::#variant_name { #(#arg_names),* }); self } } @@ -103,13 +130,209 @@ fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> Tok } }); let output = quote! { - impl XcmBuilder { + impl XcmBuilder { #(#methods)* pub fn build(self) -> Xcm { - Xcm(self.0) + Xcm(self.instructions) } } }; output } + +fn generate_builder_impl(name: &Ident, data_enum: &DataEnum) -> Result { + // We first require an instruction that load the holding register + let load_holding_variants = data_enum + .variants + .iter() + .map(|variant| { + let maybe_builder_attr = variant.attrs.iter().find(|attr| match attr.meta { + Meta::List(ref list) => { + return list.path.is_ident("builder"); + }, + _ => false, + }); + let builder_attr = match maybe_builder_attr { + Some(builder) => builder.clone(), + None => return Ok(None), /* It's not going to be an instruction that loads the + * holding register */ + }; + let Meta::List(ref list) = builder_attr.meta else { unreachable!("We checked before") }; + let inner_ident: Ident = syn::parse2(list.tokens.clone().into()).map_err(|_| { + Error::new_spanned(&builder_attr, "Expected `builder(loads_holding)`") + })?; + let ident_to_match: Ident = syn::parse_quote!(loads_holding); + if inner_ident == ident_to_match { + Ok(Some(variant)) + } else { + Err(Error::new_spanned(&builder_attr, "Expected `builder(loads_holding)`")) + } + }) + .collect::>>()?; + + let load_holding_methods = load_holding_variants + .into_iter() + .flatten() + .map(|variant| { + let variant_name = &variant.ident; + let method_name_string = &variant_name.to_string().to_snake_case(); + let method_name = syn::Ident::new(&method_name_string, variant_name.span()); + let docs = get_doc_comments(&variant); + let method = match &variant.fields { + Fields::Unnamed(fields) => { + let arg_names: Vec<_> = fields + .unnamed + .iter() + .enumerate() + .map(|(index, _)| format_ident!("arg{}", index)) + .collect(); + let arg_types: Vec<_> = fields.unnamed.iter().map(|field| &field.ty).collect(); + quote! { + #(#docs)* + pub fn #method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#variant_name(#(#arg_names),*)); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }, + Fields::Named(fields) => { + let arg_names: Vec<_> = fields.named.iter().map(|field| &field.ident).collect(); + let arg_types: Vec<_> = fields.named.iter().map(|field| &field.ty).collect(); + quote! { + #(#docs)* + pub fn #method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#variant_name { #(#arg_names),* }); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }, + _ => + return Err(Error::new_spanned( + &variant, + "Instructions that load the holding register should take operands", + )), + }; + Ok(method) + }) + .collect::, _>>()?; + + let first_impl = quote! { + impl XcmBuilder { + #(#load_holding_methods)* + } + }; + + // Then we require fees to be paid + let buy_execution_method = data_enum + .variants + .iter() + .find(|variant| variant.ident.to_string() == "BuyExecution") + .map_or( + Err(Error::new_spanned(&data_enum.variants, "No BuyExecution instruction")), + |variant| { + let variant_name = &variant.ident; + let method_name_string = &variant_name.to_string().to_snake_case(); + let method_name = syn::Ident::new(&method_name_string, variant_name.span()); + let docs = get_doc_comments(&variant); + let fields = match &variant.fields { + Fields::Named(fields) => { + let arg_names: Vec<_> = + fields.named.iter().map(|field| &field.ident).collect(); + let arg_types: Vec<_> = + fields.named.iter().map(|field| &field.ty).collect(); + quote! { + #(#docs)* + pub fn #method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#variant_name { #(#arg_names),* }); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }, + _ => + return Err(Error::new_spanned( + &variant, + "BuyExecution should have named fields", + )), + }; + Ok(fields) + }, + )?; + + let second_impl = quote! { + impl XcmBuilder { + #buy_execution_method + } + }; + + let output = quote! { + #first_impl + #second_impl + }; + + Ok(output) +} + +fn generate_builder_unpaid_impl(name: &Ident, data_enum: &DataEnum) -> Result { + let unpaid_execution_variant = data_enum + .variants + .iter() + .find(|variant| variant.ident.to_string() == "UnpaidExecution") + .ok_or(Error::new_spanned(&data_enum.variants, "No UnpaidExecution instruction"))?; + let unpaid_execution_ident = &unpaid_execution_variant.ident; + let unpaid_execution_method_name = Ident::new( + &unpaid_execution_ident.to_string().to_snake_case(), + unpaid_execution_ident.span(), + ); + let docs = get_doc_comments(&unpaid_execution_variant); + let fields = match &unpaid_execution_variant.fields { + Fields::Named(fields) => fields, + _ => + return Err(Error::new_spanned( + &unpaid_execution_variant, + "UnpaidExecution should have named fields", + )), + }; + let arg_names: Vec<_> = fields.named.iter().map(|field| &field.ident).collect(); + let arg_types: Vec<_> = fields.named.iter().map(|field| &field.ty).collect(); + Ok(quote! { + impl XcmBuilder { + #(#docs)* + pub fn #unpaid_execution_method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#unpaid_execution_ident { #(#arg_names),* }); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }) +} + +fn get_doc_comments(variant: &Variant) -> Vec { + variant + .attrs + .iter() + .filter_map(|attr| match &attr.meta { + Meta::NameValue(MetaNameValue { + value: Expr::Lit(ExprLit { lit: Lit::Str(literal), .. }), + .. + }) if attr.path().is_ident("doc") => Some(literal.value()), + _ => None, + }) + .map(|doc| syn::parse_str::(&format!("/// {}", doc)).unwrap()) + .collect() +} diff --git a/polkadot/xcm/procedural/src/lib.rs b/polkadot/xcm/procedural/src/lib.rs index 83cc6cdf98ff..7600e817d0e6 100644 --- a/polkadot/xcm/procedural/src/lib.rs +++ b/polkadot/xcm/procedural/src/lib.rs @@ -17,6 +17,7 @@ //! Procedural macros used in XCM. use proc_macro::TokenStream; +use syn::{parse_macro_input, DeriveInput}; mod builder_pattern; mod v2; @@ -56,7 +57,10 @@ pub fn impl_conversion_functions_for_junctions_v3(input: TokenStream) -> TokenSt /// .buy_execution(fees, weight_limit) /// .deposit_asset(assets, beneficiary) /// .build(); -#[proc_macro_derive(Builder)] +#[proc_macro_derive(Builder, attributes(builder))] pub fn derive_builder(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); builder_pattern::derive(input) + .unwrap_or_else(syn::Error::into_compile_error) + .into() } diff --git a/polkadot/xcm/procedural/tests/builder_pattern.rs b/polkadot/xcm/procedural/tests/builder_pattern.rs new file mode 100644 index 000000000000..eab9d67121f6 --- /dev/null +++ b/polkadot/xcm/procedural/tests/builder_pattern.rs @@ -0,0 +1,81 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test the methods generated by the Builder derive macro. +//! Tests directly on the actual Xcm struct and Instruction enum. + +use xcm::latest::prelude::*; + +#[test] +fn builder_pattern_works() { + let asset: MultiAsset = (Here, 100u128).into(); + let beneficiary: MultiLocation = AccountId32 { id: [0u8; 32], network: None }.into(); + let message: Xcm<()> = Xcm::builder() + .receive_teleported_asset(asset.clone().into()) + .buy_execution(asset.clone(), Unlimited) + .deposit_asset(asset.clone().into(), beneficiary) + .build(); + assert_eq!( + message, + Xcm(vec![ + ReceiveTeleportedAsset(asset.clone().into()), + BuyExecution { fees: asset.clone(), weight_limit: Unlimited }, + DepositAsset { assets: asset.into(), beneficiary }, + ]) + ); +} + +#[test] +fn default_builder_requires_buy_execution() { + let asset: MultiAsset = (Here, 100u128).into(); + let beneficiary: MultiLocation = AccountId32 { id: [0u8; 32], network: None }.into(); + // This is invalid, since it doesn't pay for fees. + // This is enforced by the runtime, because the build() method doesn't exist + // on the resulting type. + // let message: Xcm<()> = Xcm::builder() + // .withdraw_asset(asset.clone().into()) + // .deposit_asset(asset.into(), beneficiary) + // .build(); + + // To be able to do that, we need to use the explicitly unpaid variant + let message: Xcm<()> = Xcm::builder_unpaid() + .unpaid_execution(Unlimited, None) + .withdraw_asset(asset.clone().into()) + .deposit_asset(asset.clone().into(), beneficiary) + .build(); // This works + assert_eq!( + message, + Xcm(vec![ + UnpaidExecution { weight_limit: Unlimited, check_origin: None }, + WithdrawAsset(asset.clone().into()), + DepositAsset { assets: asset.clone().into(), beneficiary }, + ]) + ); + + // The other option doesn't have any limits whatsoever, so it should + // only be used when you really know what you're doing. + let message: Xcm<()> = Xcm::builder_unsafe() + .withdraw_asset(asset.clone().into()) + .deposit_asset(asset.clone().into(), beneficiary) + .build(); + assert_eq!( + message, + Xcm(vec![ + WithdrawAsset(asset.clone().into()), + DepositAsset { assets: asset.clone().into(), beneficiary }, + ]) + ); +} diff --git a/polkadot/xcm/procedural/tests/ui.rs b/polkadot/xcm/procedural/tests/ui.rs index a6ec35d0862a..a30f4d7dee5d 100644 --- a/polkadot/xcm/procedural/tests/ui.rs +++ b/polkadot/xcm/procedural/tests/ui.rs @@ -28,5 +28,5 @@ fn ui() { std::env::set_var("SKIP_WASM_BUILD", "1"); let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/*.rs"); + t.compile_fail("tests/ui/**/*.rs"); } diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs new file mode 100644 index 000000000000..3a103f3ddc45 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs @@ -0,0 +1,32 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when using a badly formatted attribute. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + #[builder(funds_holding = 2)] + WithdrawAsset(u128), + BuyExecution { fees: u128 }, + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr new file mode 100644 index 000000000000..978faf2e868d --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr @@ -0,0 +1,5 @@ +error: Expected `builder(loads_holding)` + --> tests/ui/builder_pattern/badly_formatted_attribute.rs:25:5 + | +25 | #[builder(funds_holding = 2)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs new file mode 100644 index 000000000000..dc5c679a96e7 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs @@ -0,0 +1,30 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when the `BuyExecution` instruction doesn't take named fields. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + BuyExecution(u128), + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr new file mode 100644 index 000000000000..dc8246770ba3 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr @@ -0,0 +1,5 @@ +error: BuyExecution should have named fields + --> tests/ui/builder_pattern/buy_execution_named_fields.rs:25:5 + | +25 | BuyExecution(u128), + | ^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs new file mode 100644 index 000000000000..070f0be6bacc --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs @@ -0,0 +1,32 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when an instruction that loads the holding register doesn't take operands. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + #[builder(loads_holding)] + WithdrawAsset, + BuyExecution { fees: u128 }, + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr new file mode 100644 index 000000000000..0358a35ad3dd --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr @@ -0,0 +1,6 @@ +error: Instructions that load the holding register should take operands + --> tests/ui/builder_pattern/loads_holding_no_operands.rs:25:5 + | +25 | / #[builder(loads_holding)] +26 | | WithdrawAsset, + | |_________________^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs new file mode 100644 index 000000000000..1ed8dd38cbad --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs @@ -0,0 +1,29 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when there's no `BuyExecution` instruction. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr new file mode 100644 index 000000000000..d8798c8223f1 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr @@ -0,0 +1,6 @@ +error: No BuyExecution instruction + --> tests/ui/builder_pattern/no_buy_execution.rs:25:5 + | +25 | / UnpaidExecution { weight_limit: (u32, u32) }, +26 | | Transact { call: Call }, + | |____________________________^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs new file mode 100644 index 000000000000..d542102d2d35 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs @@ -0,0 +1,29 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when there's no `UnpaidExecution` instruction. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + BuyExecution { fees: u128 }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr new file mode 100644 index 000000000000..c8c0748da722 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr @@ -0,0 +1,6 @@ +error: No UnpaidExecution instruction + --> tests/ui/builder_pattern/no_unpaid_execution.rs:25:5 + | +25 | / BuyExecution { fees: u128 }, +26 | | Transact { call: Call }, + | |____________________________^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs new file mode 100644 index 000000000000..5808ec571ce7 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs @@ -0,0 +1,32 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when using wrong attribute. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + #[builder(funds_holding)] + WithdrawAsset(u128), + BuyExecution { fees: u128 }, + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr new file mode 100644 index 000000000000..1ff9d1851368 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr @@ -0,0 +1,5 @@ +error: Expected `builder(loads_holding)` + --> tests/ui/builder_pattern/unexpected_attribute.rs:25:5 + | +25 | #[builder(funds_holding)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs new file mode 100644 index 000000000000..bb98d603fd91 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs @@ -0,0 +1,30 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when the `BuyExecution` instruction doesn't take named fields. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + BuyExecution { fees: u128 }, + UnpaidExecution(u32, u32), + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr new file mode 100644 index 000000000000..0a3c0a40a33b --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr @@ -0,0 +1,5 @@ +error: UnpaidExecution should have named fields + --> tests/ui/builder_pattern/unpaid_execution_named_fields.rs:26:5 + | +26 | UnpaidExecution(u32, u32), + | ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.rs similarity index 100% rename from polkadot/xcm/procedural/tests/ui/builder_pattern.rs rename to polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.rs diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.stderr similarity index 63% rename from polkadot/xcm/procedural/tests/ui/builder_pattern.stderr rename to polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.stderr index 439b40f31cae..007aa0b5ff30 100644 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern.stderr +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.stderr @@ -1,5 +1,5 @@ error: Expected the `Instruction` enum - --> tests/ui/builder_pattern.rs:23:1 + --> tests/ui/builder_pattern/wrong_target.rs:23:1 | 23 | struct SomeStruct; | ^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/src/v3/mod.rs b/polkadot/xcm/src/v3/mod.rs index 4217528f2273..bbdd504ceb0f 100644 --- a/polkadot/xcm/src/v3/mod.rs +++ b/polkadot/xcm/src/v3/mod.rs @@ -426,6 +426,7 @@ pub enum Instruction { /// Kind: *Command*. /// /// Errors: + #[builder(loads_holding)] WithdrawAsset(MultiAssets), /// Asset(s) (`assets`) have been received into the ownership of this system on the `origin` @@ -439,6 +440,7 @@ pub enum Instruction { /// Kind: *Trusted Indication*. /// /// Errors: + #[builder(loads_holding)] ReserveAssetDeposited(MultiAssets), /// Asset(s) (`assets`) have been destroyed on the `origin` system and equivalent assets should @@ -452,6 +454,7 @@ pub enum Instruction { /// Kind: *Trusted Indication*. /// /// Errors: + #[builder(loads_holding)] ReceiveTeleportedAsset(MultiAssets), /// Respond with information that the local system is expecting. @@ -776,6 +779,7 @@ pub enum Instruction { /// Kind: *Command* /// /// Errors: + #[builder(loads_holding)] ClaimAsset { assets: MultiAssets, ticket: MultiLocation }, /// Always throws an error of type `Trap`. diff --git a/polkadot/xcm/xcm-simulator/example/src/lib.rs b/polkadot/xcm/xcm-simulator/example/src/lib.rs index 03e7c19a9148..85b8ad1c5cb7 100644 --- a/polkadot/xcm/xcm-simulator/example/src/lib.rs +++ b/polkadot/xcm/xcm-simulator/example/src/lib.rs @@ -649,23 +649,4 @@ mod tests { ); }); } - - #[test] - fn builder_pattern_works() { - let asset: MultiAsset = (Here, 100u128).into(); - let beneficiary: MultiLocation = AccountId32 { id: [0u8; 32], network: None }.into(); - let message: Xcm<()> = Xcm::builder() - .withdraw_asset(asset.clone().into()) - .buy_execution(asset.clone(), Unlimited) - .deposit_asset(asset.clone().into(), beneficiary) - .build(); - assert_eq!( - message, - Xcm(vec![ - WithdrawAsset(asset.clone().into()), - BuyExecution { fees: asset.clone(), weight_limit: Unlimited }, - DepositAsset { assets: asset.into(), beneficiary }, - ]) - ); - } } diff --git a/prdoc/pr_2253.prdoc b/prdoc/pr_2253.prdoc new file mode 100644 index 000000000000..398b0a29066b --- /dev/null +++ b/prdoc/pr_2253.prdoc @@ -0,0 +1,24 @@ +# Schema: Parity PR Documentation Schema (prdoc) +# See doc at https://github.com/paritytech/prdoc + +title: Different builder pattern constructors for XCM + +doc: + - audience: Core Dev + description: | + The `builder()` constructor for XCM programs now only allows building messages that pay for fees, + i.e. messages that would pass the `AllowTopLevelPaidExecutionFrom` barrier. + Another constructor, `builder_unpaid()` requires an explicit `UnpaidExecution` instruction before + anything else. + For building messages without any restriction, `builder_unsafe` can be used. + This has been named like that since in general the other two should be used instead, but it's okay + to use it for teaching purposes or for experimenting. + +migrations: + db: [] + + runtime: [] + +crates: [] + +host_functions: [] From f5ad32e406e5108ea87f05a0b99406b80ed3226c Mon Sep 17 00:00:00 2001 From: Javier Bullrich Date: Tue, 21 Nov 2023 16:49:47 +0100 Subject: [PATCH 08/10] added action to pass review bot on merge queue (#2414) This action will trigger when a merge queue is created and will add a positive status check under the `review-bot` account, allowing the PR to pass the required check for the merge. --------- Co-authored-by: Chevdor --- .github/workflows/merge-queue.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/merge-queue.yml diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/merge-queue.yml new file mode 100644 index 000000000000..f3fb7765ca67 --- /dev/null +++ b/.github/workflows/merge-queue.yml @@ -0,0 +1,24 @@ +name: Merge-Queue + +on: + merge_group: + +jobs: + trigger-merge-queue-action: + runs-on: ubuntu-latest + environment: master + steps: + - name: Generate token + id: app_token + uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + with: + app_id: ${{ secrets.REVIEW_APP_ID }} + private_key: ${{ secrets.REVIEW_APP_KEY }} + - name: Add Merge Queue status check + uses: billyjbryant/create-status-check@3e6fa0ac599d10d9588cf9516ca4330ef669b858 # v2 + with: + authToken: ${{ steps.app_token.outputs.token }} + context: 'review-bot' + description: 'PRs for merge queue gets approved' + state: 'success' + sha: ${{ github.event.merge_group.head_commit.id }} From 50811d6b424185e1e6c0762a8b1c95db893c6750 Mon Sep 17 00:00:00 2001 From: Sophia Gold Date: Tue, 21 Nov 2023 11:34:05 -0500 Subject: [PATCH 09/10] Update tick collator for async backing (#1497) This updates the tick runtime and polkadot-parachain collator to use async backing. --- Cargo.lock | 1 + .../testing/rococo-parachain/Cargo.toml | 2 + .../testing/rococo-parachain/src/lib.rs | 47 ++++++++++++------- cumulus/polkadot-parachain/src/service.rs | 34 +++++++++----- 4 files changed, 55 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88cefe7bbef5..903e9463cf6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14530,6 +14530,7 @@ dependencies = [ "cumulus-pallet-xcm", "cumulus-pallet-xcmp-queue", "cumulus-ping", + "cumulus-primitives-aura", "cumulus-primitives-core", "cumulus-primitives-utility", "frame-benchmarking", diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml index 5e9d347a25de..08fa8b691274 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml @@ -52,6 +52,7 @@ cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system cumulus-pallet-xcm = { path = "../../../../pallets/xcm", default-features = false } cumulus-pallet-xcmp-queue = { path = "../../../../pallets/xcmp-queue", default-features = false } cumulus-ping = { path = "../../../pallets/ping", default-features = false } +cumulus-primitives-aura = { path = "../../../../primitives/aura", default-features = false } cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } cumulus-primitives-utility = { path = "../../../../primitives/utility", default-features = false } parachains-common = { path = "../../../common", default-features = false } @@ -70,6 +71,7 @@ std = [ "cumulus-pallet-xcm/std", "cumulus-pallet-xcmp-queue/std", "cumulus-ping/std", + "cumulus-primitives-aura/std", "cumulus-primitives-core/std", "cumulus-primitives-utility/std", "frame-benchmarking?/std", diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs index 6df00d43e8d3..965fb0d6adc5 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs @@ -22,13 +22,13 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; +use cumulus_pallet_parachain_system::RelayNumberMonotonicallyIncreases; use polkadot_runtime_common::xcm_sender::NoPriceForMessageDelivery; use sp_api::impl_runtime_apis; use sp_core::OpaqueMetadata; use sp_runtime::{ create_runtime_str, generic, impl_opaque_keys, - traits::{AccountIdLookup, BlakeTwo256, Block as BlockT}, + traits::{AccountIdLookup, BlakeTwo256, Block as BlockT, Hash as HashT}, transaction_validity::{TransactionSource, TransactionValidity}, ApplyExtrinsicResult, }; @@ -113,7 +113,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { state_version: 0, }; -pub const MILLISECS_PER_BLOCK: u64 = 12000; +pub const MILLISECS_PER_BLOCK: u64 = 6000; pub const SLOT_DURATION: u64 = MILLISECS_PER_BLOCK; @@ -143,18 +143,18 @@ const AVERAGE_ON_INITIALIZE_RATIO: Perbill = Perbill::from_percent(10); /// We allow `Normal` extrinsics to fill up the block up to 75%, the rest can be used /// by Operational extrinsics. const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); -/// We allow for .5 seconds of compute with a 12 second average block time. +/// We allow for 2 seconds of compute with a 6 second average block time. const MAXIMUM_BLOCK_WEIGHT: Weight = Weight::from_parts( - WEIGHT_REF_TIME_PER_SECOND.saturating_div(2), + WEIGHT_REF_TIME_PER_SECOND.saturating_mul(2), cumulus_primitives_core::relay_chain::MAX_POV_SIZE as u64, ); /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included /// into the relay chain. -const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; +const UNINCLUDED_SEGMENT_CAPACITY: u32 = 3; /// How many parachain blocks are processed by the relay chain per parent. Limits the /// number of blocks authored per slot. -const BLOCK_PROCESSING_VELOCITY: u32 = 1; +const BLOCK_PROCESSING_VELOCITY: u32 = 2; /// Relay chain slot duration, in milliseconds. const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; @@ -277,6 +277,13 @@ parameter_types! { pub const RelayOrigin: AggregateMessageOrigin = AggregateMessageOrigin::Parent; } +type ConsensusHook = cumulus_pallet_aura_ext::FixedVelocityConsensusHook< + Runtime, + RELAY_CHAIN_SLOT_DURATION_MILLIS, + BLOCK_PROCESSING_VELOCITY, + UNINCLUDED_SEGMENT_CAPACITY, +>; + impl cumulus_pallet_parachain_system::Config for Runtime { type WeightInfo = (); type RuntimeEvent = RuntimeEvent; @@ -287,13 +294,8 @@ impl cumulus_pallet_parachain_system::Config for Runtime { type ReservedDmpWeight = ReservedDmpWeight; type XcmpMessageHandler = XcmpQueue; type ReservedXcmpWeight = ReservedXcmpWeight; - type CheckAssociatedRelayNumber = RelayNumberStrictlyIncreases; - type ConsensusHook = cumulus_pallet_aura_ext::FixedVelocityConsensusHook< - Runtime, - RELAY_CHAIN_SLOT_DURATION_MILLIS, - BLOCK_PROCESSING_VELOCITY, - UNINCLUDED_SEGMENT_CAPACITY, - >; + type CheckAssociatedRelayNumber = RelayNumberMonotonicallyIncreases; + type ConsensusHook = ConsensusHook; } impl parachain_info::Config for Runtime {} @@ -584,9 +586,9 @@ impl pallet_aura::Config for Runtime { type AuthorityId = AuraId; type DisabledValidators = (); type MaxAuthorities = ConstU32<100_000>; - type AllowMultipleBlocksPerSlot = ConstBool; + type AllowMultipleBlocksPerSlot = ConstBool; #[cfg(feature = "experimental")] - type SlotDuration = pallet_aura::MinimumPeriodTimesTwo; + type SlotDuration = ConstU64; } construct_runtime! { @@ -624,7 +626,7 @@ pub type Balance = u128; /// Index of a transaction in the chain. pub type Nonce = u32; /// A hash of some data used by the chain. -pub type Hash = sp_core::H256; +pub type Hash = ::Output; /// An index to a block. pub type BlockNumber = u32; /// The address format for describing accounts. @@ -751,7 +753,7 @@ impl_runtime_apis! { impl sp_consensus_aura::AuraApi for Runtime { fn slot_duration() -> sp_consensus_aura::SlotDuration { - sp_consensus_aura::SlotDuration::from_millis(Aura::slot_duration()) + sp_consensus_aura::SlotDuration::from_millis(SLOT_DURATION) } fn authorities() -> Vec { @@ -824,6 +826,15 @@ impl_runtime_apis! { build_config::(config) } } + + impl cumulus_primitives_aura::AuraUnincludedSegmentApi for Runtime { + fn can_build_upon( + included_hash: ::Hash, + slot: cumulus_primitives_aura::Slot, + ) -> bool { + ConsensusHook::can_build_upon(included_hash, slot) + } + } } cumulus_pallet_parachain_system::register_validate_block! { diff --git a/cumulus/polkadot-parachain/src/service.rs b/cumulus/polkadot-parachain/src/service.rs index 3884fce246c3..f580835db55a 100644 --- a/cumulus/polkadot-parachain/src/service.rs +++ b/cumulus/polkadot-parachain/src/service.rs @@ -590,6 +590,7 @@ where CollatorPair, OverseerHandle, Arc>) + Send + Sync>, + Arc, ) -> Result<(), sc_service::Error>, { let parachain_config = prepare_node_config(parachain_config); @@ -723,6 +724,7 @@ where collator_key.expect("Command line arguments do not allow this. qed"), overseer_handle, announce_block, + backend.clone(), )?; } @@ -983,7 +985,8 @@ pub async fn start_rococo_parachain_node( para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + backend| { let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client)?; let proposer_factory = sc_basic_authorship::ProposerFactory::with_proof_recording( @@ -1002,11 +1005,15 @@ pub async fn start_rococo_parachain_node( client.clone(), ); - let params = BasicAuraParams { + let params = AuraParams { create_inherent_data_providers: move |_, ()| async move { Ok(()) }, block_import, - para_client: client, + para_client: client.clone(), + para_backend: backend.clone(), relay_client: relay_chain_interface, + code_hash_provider: move |block_hash| { + client.code_at(block_hash).ok().map(|c| ValidationCode::from(c).hash()) + }, sync_oracle, keystore, collator_key, @@ -1016,12 +1023,10 @@ pub async fn start_rococo_parachain_node( relay_chain_slot_duration, proposer, collator_service, - // Very limited proposal time. - authoring_duration: Duration::from_millis(500), - collation_request_receiver: None, + authoring_duration: Duration::from_millis(1500), }; - let fut = basic_aura::run::< + let fut = aura::run::< Block, sp_consensus_aura::sr25519::AuthorityPair, _, @@ -1031,6 +1036,8 @@ pub async fn start_rococo_parachain_node( _, _, _, + _, + _, >(params); task_manager.spawn_essential_handle().spawn("aura", None, fut); @@ -1376,7 +1383,8 @@ where para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + _backend| { let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client)?; let proposer_factory = sc_basic_authorship::ProposerFactory::with_proof_recording( @@ -1471,7 +1479,8 @@ where para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + _backend| { let relay_chain_interface2 = relay_chain_interface.clone(); let collator_service = CollatorService::new( @@ -1642,7 +1651,7 @@ where para_backend: backend.clone(), relay_client: relay_chain_interface, code_hash_provider: move |block_hash| { - client.code_at(block_hash).ok().map(ValidationCode).map(|c| c.hash()) + client.code_at(block_hash).ok().map(|c| ValidationCode::from(c).hash()) }, sync_oracle, keystore, @@ -1713,6 +1722,7 @@ where CollatorPair, OverseerHandle, Arc>) + Send + Sync>, + Arc, ) -> Result<(), sc_service::Error>, { let parachain_config = prepare_node_config(parachain_config); @@ -1845,6 +1855,7 @@ where collator_key.expect("Command line arguments do not allow this. qed"), overseer_handle, announce_block, + backend.clone(), )?; } @@ -1923,7 +1934,8 @@ pub async fn start_contracts_rococo_node( para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + _backend| { let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client)?; let proposer_factory = sc_basic_authorship::ProposerFactory::with_proof_recording( From 2183669d05f9b510f979a0cc3c7847707bacba2e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 21 Nov 2023 20:03:55 +0100 Subject: [PATCH 10/10] cumulus-test-service: block import fix (#2430) This is follow-up for: https://github.com/paritytech/polkadot-sdk/pull/2001 Block import queue for `test-parachain` (`cumulus-test-service`) shall use delayed best block feature. This should fixed broken zombienet tests. --- cumulus/test/service/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 6fd3e4d43d75..4d4c60d1bda6 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -211,7 +211,8 @@ pub fn new_partial( sc_service::new_full_parts::(config, None, executor)?; let client = Arc::new(client); - let block_import = ParachainBlockImport::new(client.clone(), backend.clone()); + let block_import = + ParachainBlockImport::new_with_delayed_best_block(client.clone(), backend.clone()); let registry = config.prometheus_registry();