Merge branch 'manta' into fer-change-params

.github/workflows/run_all_benchmarks.yml

@@ -20,7 +20,6 @@ jobs:
   run-benchmarks:
     timeout-minutes: 720
     runs-on: runtime-integration-test
-    if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v3
@@ -80,9 +79,10 @@ jobs:
       - name: create-chainspec
         run: |
           ${{ github.workspace }}/target/production/manta build-spec --chain $CHAIN_SPEC --disable-default-bootnode --raw > ${{ github.workspace }}/tests/data/fork.json
-      - name: append manta-pay storage
+      - name: append manta-pay and manta-sbt storage
         run: |
           wget -P ${{ github.workspace }}/tests/data https://manta-ops.s3.amazonaws.com/integration-tests-data/storage.json
+          wget -P ${{ github.workspace }}/tests/data https://mantasbt-storage.s3.amazonaws.com/mantaSbtStorage.json
           cd ${{ github.workspace }}/tests
           yarn install
           yarn

pallets/manta-sbt/Cargo.toml

@@ -85,6 +85,7 @@ precompute-coins = [
   "manta-pay/bip32",
   "manta-pay/arkworks",
   "manta-util/std",
+  "num_cpus",
   "rand_chacha",
   "tempfile",
 ]
@@ -95,6 +96,7 @@ anyhow = { version = "1.0.55", optional = true, default-features = false }
 base64 = { version = "0.20", default-features = false, features = ["alloc"] }
 indoc = { version = "2.0.1", optional = true, default-features = false }
 libsecp256k1 = { version = "0.7", default-features = false, features = ["hmac", "static-context"], optional = true }
+num_cpus = { version = "1.16.0", optional = true }
 rand_chacha = { version = "0.3.1", optional = true, default-features = false }
 sha3 = { version = "0.10.6", default-features = false }
 tempfile = { version = "3.3.0", optional = true, default-features = false }

pallets/manta-sbt/src/benchmark/mod.rs

@@ -15,30 +15,41 @@
 // along with Manta.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::{
-    benchmark::precomputed_coins::TO_PRIVATE, AccountId, Box, Call, Config, EvmAddress, Pallet,
-    Pallet as MantaSBTPallet, TransferPost,
+    AccountId, Box, Call, Config, EvmAddress, Pallet, Pallet as MantaSBTPallet, TransferPost,
 };
-use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, vec, whitelisted_caller};
+use frame_benchmarking::{benchmarks, vec, whitelisted_caller};
 use frame_support::traits::{Currency, Get};
 use frame_system::RawOrigin;
 use scale_codec::Decode;
 use sp_core::H160;
 use sp_io::hashing::keccak_256;
 
-mod precomputed_coins;
+const MINTS_OFFSET: usize = 4;
+const MINT_SIZE: usize = 553;
 
 fn alice() -> libsecp256k1::SecretKey {
     libsecp256k1::SecretKey::parse(&keccak_256(b"Alice")).unwrap()
 }
 
+fn read_mint_coins() -> &'static [u8; 22120004] {
+    core::include_bytes!("../../../../tests/data/mantaSbt_mints")
+}
+
 benchmarks! {
     where_clause {  where T::AccountId: From<AccountId> + Into<AccountId> }
     to_private {
         let caller: T::AccountId = whitelisted_caller();
         let factor = 1_000u32;
         <T as crate::Config>::Currency::make_free_balance_be(&caller, T::ReservePrice::get() * factor.into());
+        // 0..360000 asset ids have been inserted into mantasbt.
+        // so next sbt id should be 360000, or related private txs won't get passed.
+        crate::NextSbtId::<T>::put(360_000);
         Pallet::<T>::reserve_sbt(RawOrigin::Signed(caller.clone()).into(), None)?;
-        let mint_post = TransferPost::decode(&mut &*TO_PRIVATE).unwrap();
+
+        let mint_coins = read_mint_coins();
+        let mints_start = MINTS_OFFSET;
+        let to_private_coin = &mint_coins[mints_start..mints_start + MINT_SIZE];
+        let mint_post = TransferPost::decode(&mut &*to_private_coin).unwrap();
         MantaSBTPallet::<T>::new_mint_info(
             RawOrigin::Root.into(),
             0_u32.into(),
@@ -121,6 +132,7 @@ benchmarks! {
     mint_sbt_eth {
         let bab_id = 1;
         let caller: T::AccountId = whitelisted_caller();
+        crate::NextSbtId::<T>::put(360_000);
         MantaSBTPallet::<T>::change_allowlist_account(
             RawOrigin::Root.into(),
             Some(caller.clone())
@@ -139,7 +151,10 @@ benchmarks! {
             bab_id,
             bab_alice,
         )?;
-        let mint_post = TransferPost::decode(&mut &*TO_PRIVATE).unwrap();
+        let mint_coins = read_mint_coins();
+        let mints_start = MINTS_OFFSET;
+        let to_private_coin = &mint_coins[mints_start..mints_start + MINT_SIZE];
+        let mint_post = TransferPost::decode(&mut &*to_private_coin).unwrap();
 
         let signature = MantaSBTPallet::<T>::eth_sign(&alice(), &mint_post.proof, 0);
     }: mint_sbt_eth(
@@ -194,12 +209,16 @@ benchmarks! {
 
     force_to_private {
         let caller: T::AccountId = whitelisted_caller();
+        let mint_coins = read_mint_coins();
+        let mints_start = MINTS_OFFSET;
+        let to_private_coin = &mint_coins[mints_start..mints_start + MINT_SIZE];
         MantaSBTPallet::<T>::change_force_account(
             RawOrigin::Root.into(),
             Some(caller.clone())
         )?;
-        MantaSBTPallet::<T>::set_next_sbt_id(RawOrigin::Root.into(), Some(100))?;
-        let mint_post = TransferPost::decode(&mut &*TO_PRIVATE).unwrap();
+        MantaSBTPallet::<T>::set_next_sbt_id(RawOrigin::Root.into(), Some(20_000_000))?;
+        crate::NextSbtId::<T>::put(20_000_000);
+        let mint_post = TransferPost::decode(&mut &*to_private_coin).unwrap();
     }: force_to_private(
         RawOrigin::Signed(caller.clone()),
         Box::new(mint_post),
@@ -215,7 +234,12 @@ benchmarks! {
             Some(caller.clone())
         )?;
         MantaSBTPallet::<T>::set_next_sbt_id(RawOrigin::Root.into(), Some(100))?;
-        let mint_post = TransferPost::decode(&mut &*TO_PRIVATE).unwrap();
+        crate::NextSbtId::<T>::put(20_000_000);
+
+        let mint_coins = read_mint_coins();
+        let mints_start = MINTS_OFFSET;
+        let to_private_coin = &mint_coins[mints_start..mints_start + MINT_SIZE];
+        let mint_post = TransferPost::decode(&mut &*to_private_coin).unwrap();
     }: force_mint_sbt_eth(
         RawOrigin::Signed(caller.clone()),
         Box::new(mint_post),
@@ -234,5 +258,3 @@ benchmarks! {
         Some(caller)
     )
 }
-
-impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test);

pallets/manta-sbt/src/bin/precompute_coins.rs

@@ -32,9 +32,11 @@ use rand_chacha::ChaCha20Rng;
 use scale_codec::Encode;
 use std::{
     env,
-    fs::{self, OpenOptions},
+    fs::{self, File, OpenOptions},
     io::Write,
     path::PathBuf,
+    sync::Arc,
+    thread,
 };
 /// UTXO Accumulator for Building Circuits
 type UtxoAccumulator =
@@ -144,3 +146,75 @@ fn main() -> Result<()> {
     write_const_array!(target_file, TO_PRIVATE, to_private)?;
     Ok(directory.close()?)
 }
+
+#[allow(dead_code)]
+fn generate_coins_by_multithreading() -> Result<()> {
+    use std::time::Instant;
+    let now = Instant::now();
+
+    let target_file = env::args()
+        .nth(1)
+        .map(PathBuf::from)
+        .unwrap_or(env::current_dir()?.join("precomputed_coins.rs"));
+    assert!(
+        !target_file.exists(),
+        "Specify a file to place the generated files: {target_file:?}.",
+    );
+    fs::create_dir_all(
+        target_file
+            .parent()
+            .expect("This file should have a parent."),
+    )?;
+
+    let directory = tempfile::tempdir().expect("Unable to generate temporary test directory.");
+    println!("[INFO] Temporary Directory: {directory:?}");
+
+    let rng = Arc::new(ChaCha20Rng::from_seed([0; 32]));
+    let (proving_context, _, parameters, utxo_accumulator_model) =
+        load_parameters(directory.path()).expect("Unable to load parameters.");
+
+    let cpu_num = num_cpus::get() as u128;
+    let total_coins_to_be_minted = 1024;
+    let batch = total_coins_to_be_minted / cpu_num; // how many coins in one thread
+    let mut threads = vec![];
+    for i in 0..cpu_num {
+        let start = i * batch;
+        let end = batch + start;
+        let utxo = utxo_accumulator_model.clone();
+        let mut utxo_accumulator = Arc::new(UtxoAccumulator::new(utxo));
+        let mut r = rng.clone();
+        let context = Arc::new(proving_context.clone());
+        let params = Arc::new(parameters.clone());
+        let thread_join_handle = thread::spawn(move || {
+            let mut mints = Vec::new();
+            for i in start..end {
+                let asset_id = (i % 3).into();
+                println!("Iteration count: {:?}", i);
+
+                let to_private = to_private_example(
+                    &context.to_private,
+                    &params,
+                    &mut *Arc::make_mut(&mut utxo_accumulator),
+                    asset_id,
+                    1,
+                    &mut *Arc::make_mut(&mut r),
+                );
+                mints.push(to_private.clone());
+                println!("to_private size: {:?}", to_private.encode().len());
+            }
+            let path = format!("precomputed_mints-{}-to_{}", start, end);
+            let mut file = File::create(path).unwrap();
+            file.write_all(&<[TransferPost]>::encode(&mints)).unwrap();
+        });
+        threads.push(thread_join_handle);
+    }
+
+    for t in threads {
+        let _ = t.join();
+    }
+
+    let elapsed = now.elapsed();
+    println!("Elapsed: {:.2?}", elapsed);
+
+    Ok(())
+}

tests/append_storage.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const bigJson = require('big-json');
 
 // MantaPay, AssetManager and Assets storage items
 let prefixes = [
@@ -7,19 +8,37 @@ let prefixes = [
 '0xa66d1aecfdbd14d785a4d1d8723b4beb'
 ];
 
+// mantasbt, merkle trie prefix
+const mantaSbtPrefix = ['0xee3a0abfdb3bbd4914c7ac9d04e5f843'];
+
 async function main() {
   let storagePath = './data/storage.json';
+  let mantaSbtStoragePath = './data/mantaSbtStorage.json';
   let forkPath = './data/fork.json';
 
   let storage = JSON.parse(fs.readFileSync(storagePath, 'utf8'));
   let forkedSpec = JSON.parse(fs.readFileSync(forkPath, 'utf8'));
-
+  
   // Grab the items to be moved, then iterate through and insert into storage
   storage
   .filter((i) => prefixes.some((prefix) => i[0].startsWith(prefix)))
   .forEach(([key, value]) => (forkedSpec.genesis.raw.top[key] = value));
+
+  // insert mantasbt's storages into chain spec
+  if (fs.existsSync(mantaSbtStoragePath)) {
+    let mantaSbtStorage = JSON.parse(fs.readFileSync(mantaSbtStoragePath, 'utf8'));
+    mantaSbtStorage
+    .filter((i) => mantaSbtPrefix.some((prefix) => i[0].startsWith(prefix)))
+    .forEach(([key, value]) => (forkedSpec.genesis.raw.top[key] = value));
+  }
 
-  fs.writeFileSync(forkPath, JSON.stringify(forkedSpec, null, 4));
+  await new Promise((resolve, reject) => {
+    bigJson.createStringifyStream({ body: forkedSpec })
+      .pipe(fs.createWriteStream(forkPath)
+        .on('end', function() {
+          resolve();
+        }));
+  });
 
   process.exit();
 }