diff --git a/Cargo.lock b/Cargo.lock index 01a7d51dba..726a994502 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3441,6 +3441,7 @@ dependencies = [ "colored", "criterion", "indexmap 2.2.6", + "lru", "once_cell", "parking_lot", "rand", @@ -3457,6 +3458,7 @@ dependencies = [ "snarkvm-synthesizer-snark", "snarkvm-utilities", "tempfile", + "tracing", ] [[package]] diff --git a/ledger/puzzle/epoch/src/synthesis/program/mod.rs b/ledger/puzzle/epoch/src/synthesis/program/mod.rs index 89d993db80..96c8064d5d 100644 --- a/ledger/puzzle/epoch/src/synthesis/program/mod.rs +++ b/ledger/puzzle/epoch/src/synthesis/program/mod.rs @@ -103,7 +103,7 @@ function synthesize: let program = Program::from_str(&program_string)?; // Initialize a new process. - let process = Process::::load()?; + let process = Process::::for_puzzle()?; // Initialize the stack with the synthesis challenge program. let stack = Stack::new(&process, &program)?; diff --git a/synthesizer/Cargo.toml b/synthesizer/Cargo.toml index a60060fc81..85ec6337e0 100644 --- a/synthesizer/Cargo.toml +++ b/synthesizer/Cargo.toml @@ -32,7 +32,7 @@ aleo-cli = [ ] async = [ "ledger-query/async", "synthesizer-process/async" ] cuda = [ "algorithms/cuda" ] history = [ "serde" ] -rocks = [ "ledger-store/rocks" ] +rocks = ["ledger-store/rocks", "synthesizer-process/rocks"] serial = [ "console/serial", "ledger-block/serial", diff --git a/synthesizer/process/Cargo.toml b/synthesizer/process/Cargo.toml index 294a43b9bd..b67578446f 100644 --- a/synthesizer/process/Cargo.toml +++ b/synthesizer/process/Cargo.toml @@ -78,6 +78,9 @@ package = "snarkvm-ledger-store" path = "../../ledger/store" version = "=0.16.19" +[dependencies.lru] +version = "0.12" + [dependencies.synthesizer-program] package = "snarkvm-synthesizer-program" path = "../../synthesizer/program" @@ -93,9 +96,12 @@ package = "snarkvm-utilities" path = "../../utilities" version = "=0.16.19" +[dependencies.tracing] +version = "0.1" + [dependencies.aleo-std] version = "0.1.24" -default-features = false +features = ["storage"] [dependencies.colored] version = "2" diff --git a/synthesizer/process/src/deploy.rs b/synthesizer/process/src/deploy.rs index b876a37f0e..879cc12793 100644 --- a/synthesizer/process/src/deploy.rs +++ b/synthesizer/process/src/deploy.rs @@ -40,7 +40,7 @@ impl Process { /// Adds the newly-deployed program. /// This method assumes the given deployment **is valid**. #[inline] - pub fn load_deployment(&mut self, deployment: &Deployment) -> Result<()> { + pub fn load_deployment(&self, deployment: &Deployment) -> Result<()> { let timer = timer!("Process::load_deployment"); // Compute the program stack. diff --git a/synthesizer/process/src/finalize.rs b/synthesizer/process/src/finalize.rs index 5b420ac064..f2dd6d9ec8 100644 --- a/synthesizer/process/src/finalize.rs +++ b/synthesizer/process/src/finalize.rs @@ -53,7 +53,7 @@ impl Process { // Retrieve the fee stack. let fee_stack = self.get_stack(fee.program_id())?; // Finalize the fee transition. - finalize_operations.extend(finalize_fee_transition(state, store, fee_stack, fee)?); + finalize_operations.extend(finalize_fee_transition(state, store, &fee_stack, fee)?); lap!(timer, "Finalize transition for '{}/{}'", fee.program_id(), fee.function_name()); /* Finalize the deployment. */ @@ -109,7 +109,7 @@ impl Process { // Finalize the root transition. // Note that this will result in all the remaining transitions being finalized, since the number // of calls matches the number of transitions. - let mut finalize_operations = finalize_transition(state, store, stack, transition, call_graph)?; + let mut finalize_operations = finalize_transition(state, store, &stack, transition, call_graph)?; /* Finalize the fee. */ @@ -117,7 +117,7 @@ impl Process { // Retrieve the fee stack. let fee_stack = self.get_stack(fee.program_id())?; // Finalize the fee transition. - finalize_operations.extend(finalize_fee_transition(state, store, fee_stack, fee)?); + finalize_operations.extend(finalize_fee_transition(state, store, &fee_stack, fee)?); lap!(timer, "Finalize transition for '{}/{}'", fee.program_id(), fee.function_name()); } @@ -143,7 +143,7 @@ impl Process { // Retrieve the stack. let stack = self.get_stack(fee.program_id())?; // Finalize the fee transition. - let result = finalize_fee_transition(state, store, stack, fee); + let result = finalize_fee_transition(state, store, &stack, fee); finish!(timer, "Finalize transition for '{}/{}'", fee.program_id(), fee.function_name()); // Return the result. result @@ -495,7 +495,7 @@ function compute: .unwrap(); // Initialize a new process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Deploy the program. let deployment = process.deploy::(&program, rng).unwrap(); diff --git a/synthesizer/process/src/lib.rs b/synthesizer/process/src/lib.rs index 1230e96d9b..9b8bd4617f 100644 --- a/synthesizer/process/src/lib.rs +++ b/synthesizer/process/src/lib.rs @@ -64,21 +64,60 @@ use synthesizer_program::{ StackProgram, }; use synthesizer_snark::{ProvingKey, UniversalSRS, VerifyingKey}; +use tracing::{debug, info}; -use aleo_std::prelude::{finish, lap, timer}; -use indexmap::IndexMap; +use aleo_std::{ + prelude::{finish, lap, timer}, + StorageMode, +}; +use ledger_store::{ConsensusStore, TransactionStorage, TransactionStore}; +use lru::LruCache; use parking_lot::RwLock; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + num::NonZeroUsize, + sync::{Arc, Mutex}, +}; #[cfg(feature = "aleo-cli")] use colored::Colorize; +#[cfg(not(feature = "rocks"))] +use ledger_store::helpers::memory::ConsensusMemory; +#[cfg(feature = "rocks")] +use ledger_store::helpers::rocksdb::ConsensusDB; + +#[cfg(not(any(test, feature = "test")))] +const MAX_STACKS: usize = 1000; + +#[cfg(any(test, feature = "test"))] +// This must be at least 64 (same as MAX_IMPORTS) to avoid breaking test_max_imports. +const MAX_STACKS: usize = 64; + +#[cfg(feature = "rocks")] +#[derive(Clone)] +pub struct Process { + /// The universal SRS. + universal_srs: Arc>, + /// The Stack for credits.aleo + credits: Option>>, + /// The mapping of program IDs to stacks. + stacks: Arc, Arc>>>>, + /// The storage. + store: Option>>, +} + +#[cfg(not(feature = "rocks"))] #[derive(Clone)] pub struct Process { /// The universal SRS. universal_srs: Arc>, + /// The Stack for credits.aleo + credits: Option>>, /// The mapping of program IDs to stacks. - stacks: IndexMap, Arc>>, + stacks: Arc, Arc>>>>, + /// The storage. + store: Option>>, } impl Process { @@ -86,9 +125,13 @@ impl Process { #[inline] pub fn setup, R: Rng + CryptoRng>(rng: &mut R) -> Result { let timer = timer!("Process:setup"); - // Initialize the process. - let mut process = Self { universal_srs: Arc::new(UniversalSRS::load()?), stacks: IndexMap::new() }; + let mut process = Self { + universal_srs: Arc::new(UniversalSRS::load()?), + credits: None, + stacks: Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(MAX_STACKS).unwrap()))), + store: None, + }; lap!(timer, "Initialize process"); // Initialize the 'credits.aleo' program. @@ -107,7 +150,7 @@ impl Process { lap!(timer, "Synthesize credits program keys"); // Add the 'credits.aleo' stack to the process. - process.add_stack(stack); + process.credits = Some(Arc::new(stack)); finish!(timer); // Return the process. @@ -130,9 +173,13 @@ impl Process { /// Adds a new stack to the process. /// If you intend to `execute` the program, use `deploy` and `finalize_deployment` instead. #[inline] - pub fn add_stack(&mut self, stack: Stack) { + pub fn add_stack(&self, stack: Stack) { // Add the stack to the process. - self.stacks.insert(*stack.program_id(), Arc::new(stack)); + let mut lock = self.stacks.lock(); + match lock { + Ok(ref mut lock) => lock.put(*stack.program_id(), Arc::new(stack)), + Err(_) => None, + }; } } @@ -140,10 +187,36 @@ impl Process { /// Initializes a new process. #[inline] pub fn load() -> Result { + // Assumption: this is only called in test code. + Process::load_from_storage(Some(aleo_std::StorageMode::Development(0))) + } + + /// Initializes a new process. + #[inline] + pub fn load_from_storage(storage_mode: Option) -> Result { let timer = timer!("Process::load"); + debug!("Opening storage"); + let storage_mode = storage_mode.as_ref().ok_or_else(|| anyhow!("Failed to get storage mode"))?.clone(); + // try to lazy load the stack + #[cfg(feature = "rocks")] + let store = ConsensusStore::>::open(storage_mode); + #[cfg(not(feature = "rocks"))] + let store = ConsensusStore::>::open(storage_mode); + + let store = match store { + Ok(store) => store, + Err(e) => bail!("Failed to load ledger (run 'snarkos clean' and try again)\n\n{e}\n"), + }; + debug!("Opened storage"); + // Initialize the process. - let mut process = Self { universal_srs: Arc::new(UniversalSRS::load()?), stacks: IndexMap::new() }; + let mut process = Self { + universal_srs: Arc::new(UniversalSRS::load()?), + credits: None, + stacks: Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(MAX_STACKS).unwrap()))), + store: Some(store), + }; lap!(timer, "Initialize process"); // Initialize the 'credits.aleo' program. @@ -169,19 +242,39 @@ impl Process { lap!(timer, "Load circuit keys"); // Add the stack to the process. - process.add_stack(stack); + process.credits = Some(Arc::new(stack)); finish!(timer, "Process::load"); // Return the process. Ok(process) } + /// Initializes a new process without creating the 'credits.aleo' program. + #[inline] + pub fn for_puzzle() -> Result { + // Initialize the process. + let process = Self { + universal_srs: Arc::new(UniversalSRS::load()?), + credits: None, + stacks: Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(MAX_STACKS).unwrap()))), + store: None, + }; + + // Return the process. + Ok(process) + } + /// Initializes a new process without downloading the 'credits.aleo' circuit keys (for web contexts). #[inline] #[cfg(feature = "wasm")] pub fn load_web() -> Result { // Initialize the process. - let mut process = Self { universal_srs: Arc::new(UniversalSRS::load()?), stacks: IndexMap::new() }; + let mut process = Self { + universal_srs: Arc::new(UniversalSRS::load()?), + credits: None, + stacks: Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(MAX_STACKS).unwrap()))), + store: None, + }; // Initialize the 'credits.aleo' program. let program = Program::credits()?; @@ -190,7 +283,7 @@ impl Process { let stack = Stack::new(&process, &program)?; // Add the stack to the process. - process.add_stack(stack); + process.credits = Some(Arc::new(stack)); // Return the process. Ok(process) @@ -205,26 +298,64 @@ impl Process { /// Returns `true` if the process contains the program with the given ID. #[inline] pub fn contains_program(&self, program_id: &ProgramID) -> bool { - self.stacks.contains_key(program_id) + if self.credits.as_ref().map_or(false, |stack| stack.program_id() == program_id) { + return true; + } + let lock = self.stacks.lock(); + match lock { + Ok(lock) => lock.contains(program_id), + Err(_) => false, + } + } + + fn load_stack(&self, program_id: impl TryInto>) -> Result<()> { + let program_id = program_id.try_into().map_err(|_| anyhow!("Invalid program ID"))?; + info!("Lazy loading stack for {program_id}"); + let store = self.store.as_ref().ok_or_else(|| anyhow!("Failed to get store"))?; + // Retrieve the transaction store. + let transaction_store = store.transaction_store(); + let deployment_store = transaction_store.deployment_store(); + let transaction_id = deployment_store + .find_transaction_id_from_program_id(&program_id) + .map_err(|e| anyhow!("Program ID not found in storage: {e}"))? + .ok_or_else(|| anyhow!("Program ID not found in storage"))?; + let deployments = load_deployment_and_imports(self, transaction_store, transaction_id)?; + for deployment in deployments { + debug!("Loading deployment {}", deployment.0); + self.load_deployment(&deployment.1)?; + debug!("Loaded deployment"); + } + debug!("Loaded stack for {program_id}"); + Ok(()) } /// Returns the stack for the given program ID. #[inline] - pub fn get_stack(&self, program_id: impl TryInto>) -> Result<&Arc>> { + pub fn get_stack(&self, program_id: impl TryInto>) -> Result>> { // Prepare the program ID. let program_id = program_id.try_into().map_err(|_| anyhow!("Invalid program ID"))?; + // Check if the program is 'credits.aleo'. + if program_id == ProgramID::::from_str("credits.aleo")? { + return self.credits.as_ref().cloned().ok_or_else(|| anyhow!("Failed to get stack for 'credits.aleo'")); + } + // Maybe lazy load the stack + if !self.contains_program(&program_id) { + self.load_stack(program_id)?; + } // Retrieve the stack. - let stack = self.stacks.get(&program_id).ok_or_else(|| anyhow!("Program '{program_id}' does not exist"))?; + let mut lru_cache = self.stacks.lock().map_err(|_| anyhow!("Failed to lock stack"))?; + let stack = lru_cache.get(&program_id).ok_or_else(|| anyhow!("Failed to load stack"))?; // Ensure the program ID matches. ensure!(stack.program_id() == &program_id, "Expected program '{}', found '{program_id}'", stack.program_id()); // Return the stack. - Ok(stack) + Ok(stack.clone()) } /// Returns the program for the given program ID. #[inline] - pub fn get_program(&self, program_id: impl TryInto>) -> Result<&Program> { - Ok(self.get_stack(program_id)?.program()) + pub fn get_program(&self, program_id: impl TryInto>) -> Result> { + let stack = self.get_stack(program_id)?; + Ok(stack.program().clone()) } /// Returns the proving key for the given program ID and function name. @@ -288,6 +419,52 @@ impl Process { } } +// A helper function to retrieve all the deployments. +fn load_deployment_and_imports>( + process: &Process, + transaction_store: &TransactionStore, + transaction_id: N::TransactionID, +) -> Result, Deployment)>> { + // Retrieve the deployment from the transaction ID. + let deployment = match transaction_store.get_deployment(&transaction_id)? { + Some(deployment) => deployment, + None => bail!("Deployment transaction '{transaction_id}' is not found in storage."), + }; + + // Fetch the program from the deployment. + let program = deployment.program(); + let program_id = program.id(); + + // Return early if the program is already loaded. + if process.contains_program(program_id) { + return Ok(vec![]); + } + + // Prepare a vector for the deployments. + let mut deployments = vec![]; + + // Iterate through the program imports. + for import_program_id in program.imports().keys() { + // Add the imports to the process if does not exist yet. + if !process.contains_program(import_program_id) { + // Fetch the deployment transaction ID. + let Some(transaction_id) = + transaction_store.deployment_store().find_transaction_id_from_program_id(import_program_id)? + else { + bail!("Transaction ID for '{program_id}' is not found in storage."); + }; + + // Add the deployment and its imports found recursively. + deployments.extend_from_slice(&load_deployment_and_imports(process, transaction_store, transaction_id)?); + } + } + + // Once all the imports have been included, add the parent deployment. + deployments.push((*program_id, deployment)); + + Ok(deployments) +} + #[cfg(any(test, feature = "test"))] pub mod test_helpers { use super::*; diff --git a/synthesizer/process/src/tests/mod.rs b/synthesizer/process/src/tests/mod.rs index 0a1d4bbafb..348b94d872 100644 --- a/synthesizer/process/src/tests/mod.rs +++ b/synthesizer/process/src/tests/mod.rs @@ -14,3 +14,4 @@ pub mod test_credits; pub mod test_execute; +pub mod test_process; diff --git a/synthesizer/process/src/tests/test_credits.rs b/synthesizer/process/src/tests/test_credits.rs index f62fa8767a..6c1053e55d 100644 --- a/synthesizer/process/src/tests/test_credits.rs +++ b/synthesizer/process/src/tests/test_credits.rs @@ -2889,7 +2889,7 @@ mod sanity_checks { let r2 = Value::::from_str("1_500_000_000_000_000_u64").unwrap(); // Compute the assignment. - let assignment = get_assignment::<_, CurrentAleo>(stack, &private_key, function_name, &[r0, r1, r2], rng); + let assignment = get_assignment::<_, CurrentAleo>(&stack, &private_key, function_name, &[r0, r1, r2], rng); assert_eq!(16, assignment.num_public()); assert_eq!(50956, assignment.num_private()); assert_eq!(51002, assignment.num_constraints()); @@ -2917,7 +2917,7 @@ mod sanity_checks { let r1 = Value::::from_str("1_500_000_000_000_000_u64").unwrap(); // Compute the assignment. - let assignment = get_assignment::<_, CurrentAleo>(stack, &private_key, function_name, &[r0, r1], rng); + let assignment = get_assignment::<_, CurrentAleo>(&stack, &private_key, function_name, &[r0, r1], rng); assert_eq!(11, assignment.num_public()); assert_eq!(12318, assignment.num_private()); assert_eq!(12325, assignment.num_constraints()); @@ -2945,7 +2945,7 @@ mod sanity_checks { let r1 = Value::::from_str("1_500_000_000_000_000_u64").unwrap(); // Compute the assignment. - let assignment = get_assignment::<_, CurrentAleo>(stack, &private_key, function_name, &[r0, r1], rng); + let assignment = get_assignment::<_, CurrentAleo>(&stack, &private_key, function_name, &[r0, r1], rng); assert_eq!(11, assignment.num_public()); assert_eq!(12323, assignment.num_private()); assert_eq!(12330, assignment.num_constraints()); @@ -2979,7 +2979,7 @@ mod sanity_checks { let r3 = Value::::from_str(&Field::::rand(rng).to_string()).unwrap(); // Compute the assignment. - let assignment = get_assignment::<_, CurrentAleo>(stack, &private_key, function_name, &[r0, r1, r2, r3], rng); + let assignment = get_assignment::<_, CurrentAleo>(&stack, &private_key, function_name, &[r0, r1, r2, r3], rng); assert_eq!(15, assignment.num_public()); assert_eq!(38115, assignment.num_private()); assert_eq!(38151, assignment.num_constraints()); @@ -3007,7 +3007,7 @@ mod sanity_checks { let r2 = Value::::from_str(&Field::::rand(rng).to_string()).unwrap(); // Compute the assignment. - let assignment = get_assignment::<_, CurrentAleo>(stack, &private_key, function_name, &[r0, r1, r2], rng); + let assignment = get_assignment::<_, CurrentAleo>(&stack, &private_key, function_name, &[r0, r1, r2], rng); assert_eq!(12, assignment.num_public()); assert_eq!(12920, assignment.num_private()); assert_eq!(12930, assignment.num_constraints()); diff --git a/synthesizer/process/src/tests/test_execute.rs b/synthesizer/process/src/tests/test_execute.rs index cf2dcfd122..7d1ef8fcbe 100644 --- a/synthesizer/process/src/tests/test_execute.rs +++ b/synthesizer/process/src/tests/test_execute.rs @@ -18,6 +18,7 @@ use crate::{ Process, Stack, Trace, + MAX_STACKS, }; use circuit::{network::AleoV0, Aleo}; use console::{ @@ -35,12 +36,15 @@ use ledger_store::{ FinalizeStorage, FinalizeStore, }; +use lru::LruCache; use synthesizer_program::{FinalizeGlobalState, FinalizeStoreTrait, Program, StackProgram}; use synthesizer_snark::UniversalSRS; -use indexmap::IndexMap; use parking_lot::RwLock; -use std::sync::Arc; +use std::{ + num::NonZeroUsize, + sync::{Arc, Mutex}, +}; type CurrentNetwork = MainnetV0; type CurrentAleo = AleoV0; @@ -1238,7 +1242,7 @@ finalize compute: process.synthesize_key::(program.id(), &function_name, rng).unwrap(); // Reset the process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Initialize a new block store. let block_store = BlockStore::>::open(None).unwrap(); @@ -1351,7 +1355,7 @@ finalize compute: process.synthesize_key::(program.id(), &function_name, rng).unwrap(); // Reset the process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Initialize a new block store. let block_store = BlockStore::>::open(None).unwrap(); @@ -1478,7 +1482,7 @@ finalize mint_public: process.synthesize_key::(program.id(), &function_name, rng).unwrap(); // Reset the process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Initialize a new block store. let block_store = BlockStore::>::open(None).unwrap(); @@ -1607,7 +1611,7 @@ finalize mint_public: process.synthesize_key::(program0.id(), &function_name, rng).unwrap(); // Reset the process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Initialize a new block store. let block_store = BlockStore::>::open(None).unwrap(); @@ -1765,7 +1769,7 @@ finalize compute: process.synthesize_key::(program.id(), &function_name, rng).unwrap(); // Reset the process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Initialize a new block store. let block_store = BlockStore::>::open(None).unwrap(); @@ -2194,7 +2198,7 @@ finalize compute: process.synthesize_key::(program.id(), &function_name, rng).unwrap(); // Reset the process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Initialize a new block store. let block_store = BlockStore::>::open(None).unwrap(); @@ -2363,8 +2367,12 @@ fn test_process_deploy_credits_program() { let rng = &mut TestRng::default(); // Initialize an empty process without the `credits` program. - let empty_process = - Process { universal_srs: Arc::new(UniversalSRS::::load().unwrap()), stacks: IndexMap::new() }; + let empty_process = Process { + universal_srs: Arc::new(UniversalSRS::::load().unwrap()), + credits: None, + stacks: Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(MAX_STACKS).unwrap()))), + store: None, + }; // Construct the process. let process = Process::load().unwrap(); @@ -2421,7 +2429,7 @@ function {function_name}: .unwrap(); // Reset the process. - let mut process = Process::load().unwrap(); + let process = Process::load().unwrap(); // Initialize a new block store. let block_store = BlockStore::>::open(None).unwrap(); diff --git a/synthesizer/process/src/tests/test_process.rs b/synthesizer/process/src/tests/test_process.rs new file mode 100644 index 0000000000..87b0d89ebd --- /dev/null +++ b/synthesizer/process/src/tests/test_process.rs @@ -0,0 +1,92 @@ +// Copyright (C) 2019-2023 Aleo Systems Inc. +// This file is part of the snarkVM library. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::str::FromStr; + +use console::{ + network::MainnetV0, + program::{Parser, ProgramID}, +}; +use synthesizer_program::Program; + +use crate::Process; + +type CurrentNetwork = MainnetV0; + +#[test] +pub fn test_credits() { + let process = Process::load().unwrap(); + let credits_id = ProgramID::::from_str("credits.aleo").unwrap(); + assert!(process.contains_program(&credits_id)); +} + +#[test] +pub fn test_cache() { + let (_, program1) = Program::::parse( + r" +program testing1.aleo; + +function compute: + input r0 as u32.private; + input r1 as u32.public; + add r0 r1 into r2; + output r2 as u32.public;", + ) + .unwrap(); + // Initialize a new process. + let process = crate::test_helpers::sample_process(&program1); + assert!(process.contains_program(program1.id())); +} + +#[test] +pub fn test_cache_evict() { + let (_, program1) = Program::::parse( + r" +program testing1.aleo; + +function compute: + input r0 as u32.private; + input r1 as u32.public; + add r0 r1 into r2; + output r2 as u32.public;", + ) + .unwrap(); + // Initialize a new process. + let mut process = crate::test_helpers::sample_process(&program1); + assert!(process.contains_program(program1.id())); + + for i in 2..=65 { + let source = format!( + r" +program testing{i}.aleo; + +function compute: + input r0 as u32.private; + input r1 as u32.public; + add r0 r1 into r2; + output r2 as u32.public;" + ); + // Program1 should still be cached + assert!(process.contains_program(program1.id())); + let (_, program) = Program::::parse(&source).unwrap(); + process.add_program(&program).unwrap(); + assert!(process.contains_program(program.id())); + } + + // only 64 programs are cached, so program1 should be evicted + assert!(!process.contains_program(program1.id())); + // test we still have credits.aleo + let credits_id = ProgramID::::from_str("credits.aleo").unwrap(); + assert!(process.contains_program(&credits_id)); +} diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index 1ab759736b..59c5cb143b 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -570,7 +570,7 @@ impl> VM { // Acquire the write lock on the process. // Note: Due to the highly-sensitive nature of processing all `finalize` calls, // we choose to acquire the write lock for the entire duration of this atomic batch. - let mut process = self.process.write(); + let process = self.process.write(); // Initialize a list for the deployed stacks. let mut stacks = Vec::new(); diff --git a/synthesizer/src/vm/mod.rs b/synthesizer/src/vm/mod.rs index 6f6565a358..12cf2f7592 100644 --- a/synthesizer/src/vm/mod.rs +++ b/synthesizer/src/vm/mod.rs @@ -54,7 +54,6 @@ use ledger_store::{ ConsensusStore, FinalizeMode, FinalizeStore, - TransactionStorage, TransactionStore, TransitionStore, }; @@ -96,7 +95,7 @@ impl> VM { #[inline] pub fn from(store: ConsensusStore) -> Result { // Initialize a new process. - let mut process = Process::load()?; + let process = Process::load_from_storage(Some(store.storage_mode().clone()))?; // Initialize the store for 'credits.aleo'. let credits = Program::::credits()?; @@ -108,83 +107,6 @@ impl> VM { } } - // A helper function to retrieve all the deployments. - fn load_deployment_and_imports>( - process: &Process, - transaction_store: &TransactionStore, - transaction_id: N::TransactionID, - ) -> Result, Deployment)>> { - // Retrieve the deployment from the transaction ID. - let deployment = match transaction_store.get_deployment(&transaction_id)? { - Some(deployment) => deployment, - None => bail!("Deployment transaction '{transaction_id}' is not found in storage."), - }; - - // Fetch the program from the deployment. - let program = deployment.program(); - let program_id = program.id(); - - // Return early if the program is already loaded. - if process.contains_program(program_id) { - return Ok(vec![]); - } - - // Prepare a vector for the deployments. - let mut deployments = vec![]; - - // Iterate through the program imports. - for import_program_id in program.imports().keys() { - // Add the imports to the process if does not exist yet. - if !process.contains_program(import_program_id) { - // Fetch the deployment transaction ID. - let Some(transaction_id) = - transaction_store.deployment_store().find_transaction_id_from_program_id(import_program_id)? - else { - bail!("Transaction ID for '{program_id}' is not found in storage."); - }; - - // Add the deployment and its imports found recursively. - deployments.extend_from_slice(&load_deployment_and_imports( - process, - transaction_store, - transaction_id, - )?); - } - } - - // Once all the imports have been included, add the parent deployment. - deployments.push((*program_id, deployment)); - - Ok(deployments) - } - - // Retrieve the transaction store. - let transaction_store = store.transaction_store(); - // Retrieve the list of deployment transaction IDs. - let deployment_ids = transaction_store.deployment_transaction_ids().collect::>(); - // Load the deployments from the store. - for (i, chunk) in deployment_ids.chunks(256).enumerate() { - debug!( - "Loading deployments {}-{} (of {})...", - i * 256, - ((i + 1) * 256).min(deployment_ids.len()), - deployment_ids.len() - ); - let deployments = cfg_iter!(chunk) - .map(|transaction_id| { - // Load the deployment and its imports. - load_deployment_and_imports(&process, transaction_store, **transaction_id) - }) - .collect::>>()?; - - for (program_id, deployment) in deployments.iter().flatten() { - // Load the deployment if it does not exist in the process yet. - if !process.contains_program(program_id) { - process.load_deployment(deployment)?; - } - } - } - // Return the new VM. Ok(Self { process: Arc::new(RwLock::new(process)), diff --git a/vm/package/build.rs b/vm/package/build.rs index d7057df8e6..b7d55576c3 100644 --- a/vm/package/build.rs +++ b/vm/package/build.rs @@ -177,11 +177,8 @@ impl Package { let process = self.get_process()?; // Retrieve the imported programs. - let imported_programs = program - .imports() - .keys() - .map(|program_id| process.get_program(program_id).cloned()) - .collect::>>()?; + let imported_programs = + program.imports().keys().map(|program_id| process.get_program(program_id)).collect::>>()?; // Synthesize each proving and verifying key. for function_name in program.functions().keys() { @@ -230,7 +227,7 @@ impl Package { CallOperator::Locator(locator) => { (process.get_program(locator.program_id())?, locator.resource()) } - CallOperator::Resource(resource) => (program, resource), + CallOperator::Resource(resource) => (program.clone(), resource), }; // If this is a function call, save its corresponding prover and verifier files. if program.contains_function(resource) { diff --git a/vm/package/execute.rs b/vm/package/execute.rs index 190ef66199..255777d5b0 100644 --- a/vm/package/execute.rs +++ b/vm/package/execute.rs @@ -60,7 +60,7 @@ impl Package { // Retrieve the program and resource. let (program, resource) = match call.operator() { CallOperator::Locator(locator) => (process.get_program(locator.program_id())?, locator.resource()), - CallOperator::Resource(resource) => (program, resource), + CallOperator::Resource(resource) => (program.clone(), resource), }; // If this is a function call, save its corresponding prover and verifier files. if program.contains_function(resource) {