Lazy loading (#2840)

* feat: implement lazy loading backend and call executor experiments * code cleanup * add lazy loading chain spec * feat(lazy-loading): replace async-std with tokio and retry failed requests * remove local polkadot-sdk dependencies * fix Cargo.toml * chore: update Cargo.lock file * fix editor config errors * fix editor config errors * fix build * fix lazy-loading feature * cleanup * reset AsyncBacking > SlotInfo * test: add lazy loading tests * remove parent field * remove files * test: fix formatting * fix(ci): lazy loading test * fix test * fix format * test: move lazy loading state overrides to configs folder * fix build and add mandatory storage override * fix pipeline * add state overrides for dev accounts * fix build * remove redundant test * fix remark from review * improve lazy loading test
moonbeam-foundation · Sep 11, 2024 · 30e29c4 · 30e29c4
1 parent 9c821e2
commit 30e29c4
Show file tree

Hide file tree

Showing 25 changed files with 5,176 additions and 78 deletions.
diff --git a/.github/workflow-templates/cargo-build/action.yml b/.github/workflow-templates/cargo-build/action.yml
@@ -48,6 +48,8 @@ runs:
           params="$params --features ${{ inputs.features }}"
         fi
         echo "cargo build $params"
+        cargo build $params --features lazy-loading
+        cp target/release/moonbeam target/release/lazy-loading
         cargo build $params
     - name: Display binary comments
       shell: bash
@@ -74,3 +76,4 @@ runs:
       run: |
         mkdir -p build
         cp target/release/moonbeam build/moonbeam;
+        cp target/release/lazy-loading build/lazy-loading;
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -700,6 +700,63 @@ jobs:
             org.opencontainers.image.revision=${{ github.sha }}
             org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }}
 
+  lazy-loading-tests:
+    runs-on:
+      labels: bare-metal
+    needs: ["set-tags", "build"]
+    strategy:
+      fail-fast: false
+      matrix:
+        chain: ["moonbeam"]
+    env:
+      GH_WORKFLOW_MATRIX_CHAIN: ${{ matrix.chain }}
+      DEBUG_COLORS: 1
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.set-tags.outputs.git_ref }}
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 8
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20.10.0
+      - name: Create local folders
+        run: |
+          mkdir -p target/release/wbuild/${{ matrix.chain }}-runtime/
+          mkdir -p test/tmp
+      - name: "Download branch built runtime"
+        uses: actions/download-artifact@v4
+        with:
+          name: runtimes
+          path: target/release/wbuild/${{ matrix.chain }}-runtime/
+      - name: "Download branch built node"
+        uses: actions/download-artifact@v4
+        with:
+          name: moonbeam
+          path: target/release
+      - name: "Run lazy loading tests"
+        run: |
+          cd test
+          pnpm install
+          chmod uog+x ../target/release/lazy-loading
+          pnpm moonwall test lazy_loading_${{ matrix.chain }}
+      - name: Zip and Upload Node Logs on Failure
+        if: failure()
+        run: |
+          TIMESTAMP=$(date +%Y%m%d%H%M%S)
+          export NODE_LOGS_ZIP="node_logs_$TIMESTAMP.zip"
+          MOST_RECENT_ZOMBIE_DIR=$(ls -td /tmp/zombie-* | head -n 1)
+          find $MOST_RECENT_ZOMBIE_DIR -maxdepth 1 -type f -name '*.log' -exec zip -r $NODE_LOGS_ZIP {} \;
+          echo "NODE_LOGS_ZIP=${NODE_LOGS_ZIP}" >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: failed-node-logs
+          path: ${{ env.NODE_LOGS_ZIP }}
+
   chopsticks-upgrade-test:
     runs-on:
       labels: bare-metal

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -224,13 +224,15 @@ sp-consensus = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", br
 sp-storage = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 sp-timestamp = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 sp-wasm-interface = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
+sp-rpc = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 substrate-build-script-utils = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 substrate-frame-rpc-system = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 substrate-prometheus-endpoint = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 substrate-test-client = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 substrate-test-runtime = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 substrate-test-runtime-client = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 substrate-wasm-builder = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
+substrate-rpc-client = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
 
 # Frontier (wasm)
 
@@ -404,10 +406,12 @@ schnorrkel = { version = "0.11.4", default-features = false, features = [
 	"preaudit_deprecated",
 ] }
 tokio = { version = "1.36" }
+tokio-retry = { version = "0.3.0" }
 tracing = "0.1.34"
 tracing-core = "0.1.29"
 trie-root = "0.15.2"
 url = "2.2.2"
+thiserror = "1.0.63"
 
 # The list of dependencies below (which can be both direct and indirect dependencies) are crates
 # that are suspected to be CPU-intensive, and that are unlikely to require debugging (as some of
@@ -493,4 +497,4 @@ inherits = "release"
 overflow-checks = true
 
 [patch."https://github.com/paritytech/polkadot-sdk"]
-sp-crypto-ec-utils = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
+sp-crypto-ec-utils = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" }
diff --git a/node/Cargo.toml b/node/Cargo.toml
@@ -42,6 +42,8 @@ moonriver-native = [ "moonbeam-cli/moonriver-native", "moonbeam-service/moonrive
 
 metadata-hash = ["moonbeam-service/metadata-hash"]
 
+lazy-loading = ["moonbeam-service/lazy-loading", "moonbeam-cli/lazy-loading"]
+
 test-spec = []
 
 runtime-benchmarks = [

diff --git a/node/cli-opt/src/lib.rs b/node/cli-opt/src/lib.rs
@@ -11,8 +11,10 @@
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU General Public License for more details.
 
+use std::path::PathBuf;
 // You should have received a copy of the GNU General Public License
 // along with Moonbeam.  If not, see <http://www.gnu.org/licenses/>.
+use primitive_types::H256;
 use std::str::FromStr;
 
 pub mod account_key;
@@ -110,3 +112,11 @@ pub struct RpcConfig {
 	pub frontier_backend_config: FrontierBackendConfig,
 	pub no_prometheus_prefix: bool,
 }
+
+#[derive(Clone)]
+pub struct LazyLoadingConfig {
+	pub state_rpc: String,
+	pub from_block: Option<H256>,
+	pub state_overrides_path: Option<PathBuf>,
+	pub runtime_override: Option<PathBuf>,
+}
diff --git a/node/cli/Cargo.toml b/node/cli/Cargo.toml
@@ -54,6 +54,8 @@ default = [
     "westend-native",
 ]
 
+lazy-loading = ["sc-service/test-helpers", "moonbeam-service/lazy-loading"]
+
 westend-native = ["polkadot-service/westend-native"]
 
 moonbase-native = ["moonbeam-service/moonbase-native", "westend-native"]

diff --git a/node/cli/src/cli.rs b/node/cli/src/cli.rs
@@ -26,6 +26,12 @@ use sc_cli::{Error as CliError, SubstrateCli};
 use std::path::PathBuf;
 use std::time::Duration;
 
+#[cfg(feature = "lazy-loading")]
+fn parse_block_hash(s: &str) -> Result<sp_core::H256, String> {
+	use std::str::FromStr;
+	sp_core::H256::from_str(s).map_err(|err| err.to_string())
+}
+
 /// Sub-commands supported by the collator.
 #[derive(Debug, clap::Subcommand)]
 pub enum Subcommand {
@@ -136,6 +142,22 @@ pub struct RunCmd {
 	#[clap(long)]
 	pub dev_service: bool,
 
+	#[cfg(feature = "lazy-loading")]
+	#[clap(long)]
+	pub fork_chain_from_rpc: Option<String>,
+
+	#[cfg(feature = "lazy-loading")]
+	#[arg(long, value_name = "BLOCK", value_parser = parse_block_hash)]
+	pub block: Option<sp_core::H256>,
+
+	#[cfg(feature = "lazy-loading")]
+	#[clap(long, value_name = "PATH", value_parser)]
+	pub fork_state_overrides: Option<PathBuf>,
+
+	#[cfg(feature = "lazy-loading")]
+	#[clap(long, value_name = "PATH", value_parser)]
+	pub runtime_override: Option<PathBuf>,
+
 	/// When blocks should be sealed in the dev service.
 	///
 	/// Options are "instant", "manual", or timer interval in milliseconds