Allow async backing in moonbase (#2593)

* upgrade to polkadot v1.3.0

* Temporarily disable kusama and polkadot runtime dependencies

* Remove support for Kusama Native Runtime

* Update evm pin

* update deposit_asset function signature

* Add Hasher generic parameter to Bounded

* Rename InvalidEvmTransactionError to TransactionValidationError

* Use transfer_allow_death instead of deprecated transfer

* fix build issue

* Use pallet_xcm::RuntimeCall

* BackendReader renamed to Backend in Frontier

* Update frontier pin

* Temporarily update new missing weight functions

* Adapt to new return type of ValidatedTransaction::apply

* IdentityInfo in pallet_identity is now generic

* Associated type Hasher for QueryPreimage, StorePreimage and Bounded

* Update Frontier Pin

* fix runtime compilation errors

* Rename BackendReader to Backend

* Remove Send and Sync Trait Bounds for Frontier Backend

* Implement closest_merkle_tree and child_closest_merkle_tree for StorageProvider

* Set block_relay to none

* Use polkadot_primitives v6

* Annotate EC for Eth::new

* Update polkadot-sdk pin

* Use transferAllowDeath instead of transfer in tests

* Update polkadot-sdk pin

* Use transferAllowDeath instead of transfer

* fix typescript test

* fix typescript tests

* Fix zombie tests

* use async backing moonkit branch

* implement async-backing for moonbase

* refactor client part top be compatible with the new nimbus api

* update moonkit pin

* impl consensus hook in runtimes

* update moonkit

* enable feature parameterized-consensus-hook globally

* Fix test balance Transferable

* Temporarily disabled relay-encoder rust tests

* Fix Rust Tests build

* update moonkit

* add cli option --experimental-async-backing

* fmt

* add pallet async-backing to moonbase

* update polkadot-sdk fork

* update polkadot-sdk fork

* improve relay proof mock

* Fix build with runtime-benchmarks

* fmt

* Typescript Fmt

* Set MaxHolds to 1

* comply editorconfig

* fix democracy TS tests

* fix Eslint warning

* Fix typescript test

* fix txpool TS test

* fix precompile identity test

* remove unused import

* editorconfig

* Fix rust tests

* Fix rust tests (no reserve anymore with preimage)

* Fix xcm rust tests

* reimplement create_inherent_data_providers

* editorconfig

* compile rust tests

* addn rust feature experimental-async-backing

* Fix build error

* update moonkit pin

* Update polkadot-sdk pin

* fix collective test

* Add Evm Fix

* Update evm pin

* change moonkit branch

* simplify idty precompile trait bounds

* update moonkit pin

* use relay slot for evm timestamp and precompiles permit time

* use our own impl to compute a reliable timestamp (moonbase only)

* improve pov test

* remove useless function set_parachain_inherent_data

* improve pov precompile test

* update polkadot-sdk and moonkit forks

* Inject relay timestamp in relay proof and use it as blockchain timestamp

* fix rust compilation

* cleanup code

* verify if the cause is really MinimumPeriod

* fix rust tests

* reorder moonbase runtime imports

* Allow same relay number for several parablocks

---------

Co-authored-by: Ahmad Kaouk <[email protected]>
Co-authored-by: Agusrodri <[email protected]>

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-exclude = [ "bin/utils/moonkey" ]
+exclude = ["bin/utils/moonkey"]
 members = [
 	"bin/utils/moonkey",
 	"client/rpc/finality",
@@ -40,7 +40,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-authors = [ "PureStake" ]
+authors = ["PureStake"]
 repository = "https://github.com/PureStake/moonbeam"
 
 [workspace.dependencies]
@@ -158,6 +158,7 @@ sp-api = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch =
 sp-application-crypto = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 sp-block-builder = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 sp-consensus-babe = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
+sp-consensus-slots = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 sp-core = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 sp-debug-derive = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 sp-externalities = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
@@ -258,7 +259,9 @@ fp-storage = { git = "https://github.com/moonbeam-foundation/frontier", branch =
 
 # Cumulus (wasm)
 cumulus-pallet-dmp-queue = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
-cumulus-pallet-parachain-system = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
+cumulus-pallet-parachain-system = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false, features = [
+	"parameterized-consensus-hook",
+] }
 cumulus-pallet-xcm = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 cumulus-pallet-xcmp-queue = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 cumulus-primitives-core = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
@@ -272,6 +275,7 @@ parachain-info = { git = "https://github.com/moonbeam-foundation/polkadot-sdk",
 cumulus-client-cli = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
 cumulus-client-collator = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
 cumulus-client-consensus-common = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
+cumulus-client-consensus-proposer = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
 cumulus-client-consensus-relay-chain = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
 cumulus-client-network = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
 cumulus-client-service = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
@@ -282,8 +286,8 @@ cumulus-relay-chain-rpc-interface = { git = "https://github.com/moonbeam-foundat
 
 # Polkadot / XCM (wasm)
 orml-traits = { git = "https://github.com/moonbeam-foundation/open-runtime-module-library", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
-orml-xcm-support = { git = "https://github.com/moonbeam-foundation/open-runtime-module-library", branch = "moonbeam-polkadot-v1.3.0", default-features = false  }
-orml-xtokens = { git = "https://github.com/moonbeam-foundation/open-runtime-module-library", branch = "moonbeam-polkadot-v1.3.0", default-features = false  }
+orml-xcm-support = { git = "https://github.com/moonbeam-foundation/open-runtime-module-library", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
+orml-xtokens = { git = "https://github.com/moonbeam-foundation/open-runtime-module-library", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 pallet-xcm = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 pallet-xcm-benchmarks = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 polkadot-core-primitives = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
@@ -305,8 +309,10 @@ westend-runtime = { git = "https://github.com/moonbeam-foundation/polkadot-sdk",
 xcm-simulator = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.3.0" }
 
 # Moonkit (wasm)
+async-backing-primitives = { git = "https://github.com/Moonsong-Labs/moonkit", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 moonkit-xcm-primitives = { package = "xcm-primitives", git = "https://github.com/Moonsong-Labs/moonkit", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 nimbus-primitives = { git = "https://github.com/Moonsong-Labs/moonkit", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
+pallet-async-backing = { git = "https://github.com/Moonsong-Labs/moonkit", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 pallet-author-inherent = { git = "https://github.com/Moonsong-Labs/moonkit", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 pallet-author-mapping = { git = "https://github.com/Moonsong-Labs/moonkit", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
 pallet-author-slot-filter = { git = "https://github.com/Moonsong-Labs/moonkit", branch = "moonbeam-polkadot-v1.3.0", default-features = false }
@@ -324,7 +330,9 @@ async-trait = { version = "0.1.42" }
 blake2-rfc = { version = "0.2.18", default-features = false }
 derive_more = "0.99"
 environmental = { version = "1.1.2", default-features = false }
-frame-metadata = { version = "16.0.0", default-features = false, features = ["current"] }
+frame-metadata = { version = "16.0.0", default-features = false, features = [
+	"current",
+] }
 hex = { version = "0.4.3", default-features = false }
 hex-literal = { version = "0.3.4" }
 impl-serde = { version = "0.3.1", default-features = false }
@@ -339,17 +347,15 @@ serde = { version = "1.0.101", default-features = false }
 sha3 = { version = "0.10", default-features = false }
 slices = "0.2.0"
 smallvec = "1.8.0"
-strum = { version = "0.24", default-features = false, features = [ "derive" ] }
+strum = { version = "0.24", default-features = false, features = ["derive"] }
 strum_macros = "0.24"
 
 # Other (client)
 ansi_term = "0.12.1"
 assert_cmd = "2.0.10"
 async-io = "1.3"
-bip32 = { version = "0.5.1", default-features = false, features = [
-	"bip39",
-] }
-clap = { version = "4.0.9", features = [ "derive" ] }
+bip32 = { version = "0.5.1", default-features = false, features = ["bip39"] }
+clap = { version = "4.0.9", features = ["derive"] }
 exit-future = "0.2"
 flume = "0.10.9"
 futures = { version = "0.3.21" }
@@ -451,7 +457,7 @@ opt-level = 3
 panic = "unwind"
 
 [profile.testnet]
-debug = 1 # debug symbols are useful for profilers
+debug = 1               # debug symbols are useful for profilers
 debug-assertions = true
 inherits = "release"
 overflow-checks = true

core-primitives/Cargo.toml

@@ -9,8 +9,8 @@ version = "0.1.1"
 
 [dependencies]
 account = { workspace = true }
-
 fp-self-contained = { workspace = true }
+hex-literal = { workspace = true }
 sp-core = { workspace = true }
 sp-runtime = { workspace = true }
 

core-primitives/src/lib.rs

@@ -49,3 +49,10 @@ pub type Address = AccountId;
 pub type AccountIndex = u32;
 /// Digest item type.
 pub type DigestItem = generic::DigestItem;
+
+pub mod well_known_relay_keys {
+	use hex_literal::hex;
+
+	pub const TIMESTAMP_NOW: &[u8] =
+		&hex!["f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb"];
+}

node/cli/src/cli.rs

@@ -232,6 +232,11 @@ pub struct RunCmd {
 	/// Removes moonbeam prefix from Prometheus metrics
 	#[clap(long)]
 	pub no_prometheus_prefix: bool,
+
+	/// Enable asynchronous backing (for collators only)
+	/// This feature is still experimental, please don't use it in production.
+	#[clap(hide = true, long = "experimental-async-backing")]
+	pub async_backing: bool,
 }
 
 impl RunCmd {

node/cli/src/command.rs

@@ -791,6 +791,7 @@ pub fn run() -> Result<()> {
 						collator_options,
 						id,
 						rpc_config,
+						cli.run.async_backing,
 						hwbench,
 					)
 					.await
@@ -806,6 +807,7 @@ pub fn run() -> Result<()> {
 						collator_options,
 						id,
 						rpc_config,
+						cli.run.async_backing,
 						hwbench,
 					)
 					.await
@@ -821,6 +823,7 @@ pub fn run() -> Result<()> {
 						collator_options,
 						id,
 						rpc_config,
+						cli.run.async_backing,
 						hwbench,
 					)
 					.await

node/service/Cargo.toml

@@ -115,9 +115,11 @@ pallet-ethereum = { workspace = true, features = [
 ] }
 
 # Cumulus / Nimbus
+async-backing-primitives = { workspace = true }
 cumulus-client-cli = { workspace = true }
 cumulus-client-collator = { workspace = true }
 cumulus-client-consensus-common = { workspace = true }
+cumulus-client-consensus-proposer = { workspace = true }
 cumulus-client-consensus-relay-chain = { workspace = true }
 cumulus-client-network = { workspace = true }
 cumulus-client-service = { workspace = true }

node/service/src/client.rs

@@ -46,6 +46,7 @@ pub trait RuntimeApiCollection:
 	+ nimbus_primitives::NimbusApi<Block>
 	+ cumulus_primitives_core::CollectCollationInfo<Block>
 	+ session_keys_primitives::VrfApi<Block>
+	+ async_backing_primitives::UnincludedSegmentApi<Block>
 {
 }
 
@@ -65,6 +66,7 @@ impl<Api> RuntimeApiCollection for Api where
 		+ nimbus_primitives::NimbusApi<Block>
 		+ cumulus_primitives_core::CollectCollationInfo<Block>
 		+ session_keys_primitives::VrfApi<Block>
+		+ async_backing_primitives::UnincludedSegmentApi<Block>
 {
 }