From c92f477139143597a939732fc5b35478b300ce9f Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 7 Feb 2024 15:57:20 +1300 Subject: [PATCH] feat: Add "StartBlock::Continue" to registry contract/types (#548) The start block parameter (`start_block_height`) has two states, signifying "Start from block" and "Start from latest" only. With the removal of the continuous "real-time" process, which essentially runs Indexers "From Interruption", the need for a third state arises, representing "Interruption" arises. This PR expands the registry contract/types to accommodate this change. This work was a good opportunity to refactor the existing types, removing unnecessary fields and tailoring it to the new architecture. ## Registry Types Refactoring - `filter: IndexerRule` -> `rule: Rule`: `IndexerRule` contained a lot of noise/data which was not needed. `id`, `name`, and `indexer_rule_kind` have all been removed. The only useful part here is `MatchingRule` which has been renamed to just `Rule`. - `schema` is now non-optional: This field is always required so it makes sense to convey that in the code. Having it as an `Option` created lots of unnecessary checks throughout the system. - `start_block_height` replaced by `start_block`: The latter being an `enum` to represent "From Latest", "From Block", and "From Interruption" ## Public methods/API To minimise the disruption of the existing contract consumers, the public API remains unchanged, i.e. the core methods (`list_indexer_functions`, `register_indexer_function`, etc.) use/return the same types. As the underlying data will be migrated to the new types, these methods infer the old types from the new ones. New methods have been created to work with the new types (`register`, `list_all`, etc.) allowing clients to move over when possible, as opposed to creating a breaking change. --- block-streamer/src/block_stream.rs | 2 +- block-streamer/src/indexer_config.rs | 2 +- block-streamer/src/rules/mod.rs | 2 +- block-streamer/src/rules/outcomes_reducer.rs | 2 +- .../src/server/block_streamer_service.rs | 2 +- coordinator/src/main.rs | 2 +- coordinator/src/migration.rs | 6 +- coordinator/src/registry.rs | 6 +- registry/contract/src/lib.rs | 634 ++++++++++++------ registry/types/src/lib.rs | 189 +++++- 10 files changed, 628 insertions(+), 219 deletions(-) diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs index fd98e03cd..a605f30c4 100644 --- a/block-streamer/src/block_stream.rs +++ b/block-streamer/src/block_stream.rs @@ -294,7 +294,7 @@ mod tests { ) .unwrap(), function_name: "test".to_string(), - indexer_rule: registry_types::IndexerRule { + indexer_rule: registry_types::OldIndexerRule { indexer_rule_kind: registry_types::IndexerRuleKind::Action, matching_rule: registry_types::MatchingRule::ActionAny { affected_account_id: "queryapi.dataplatform.near".to_string(), diff --git a/block-streamer/src/indexer_config.rs b/block-streamer/src/indexer_config.rs index aebd7bb2c..435c4aa3b 100644 --- a/block-streamer/src/indexer_config.rs +++ b/block-streamer/src/indexer_config.rs @@ -2,7 +2,7 @@ use near_lake_framework::near_indexer_primitives::types::AccountId; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -use registry_types::IndexerRule; +use registry_types::OldIndexerRule as IndexerRule; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct IndexerConfig { diff --git a/block-streamer/src/rules/mod.rs b/block-streamer/src/rules/mod.rs index 5c5058514..f87b906fd 100644 --- a/block-streamer/src/rules/mod.rs +++ b/block-streamer/src/rules/mod.rs @@ -3,7 +3,7 @@ pub mod outcomes_reducer; pub mod types; use near_lake_framework::near_indexer_primitives::StreamerMessage; -use registry_types::{IndexerRule, MatchingRule}; +use registry_types::{MatchingRule, OldIndexerRule as IndexerRule}; use types::{ChainId, IndexerRuleMatch}; diff --git a/block-streamer/src/rules/outcomes_reducer.rs b/block-streamer/src/rules/outcomes_reducer.rs index 934eda37c..e66fab593 100644 --- a/block-streamer/src/rules/outcomes_reducer.rs +++ b/block-streamer/src/rules/outcomes_reducer.rs @@ -115,7 +115,7 @@ fn build_indexer_rule_match_payload( #[cfg(test)] mod tests { - use registry_types::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; + use registry_types::{IndexerRuleKind, MatchingRule, OldIndexerRule as IndexerRule, Status}; use crate::rules::outcomes_reducer::reduce_indexer_rule_matches_from_outcomes; use crate::rules::types::{ChainId, IndexerRuleMatch}; diff --git a/block-streamer/src/server/block_streamer_service.rs b/block-streamer/src/server/block_streamer_service.rs index 829ec4876..2fbb6aee5 100644 --- a/block-streamer/src/server/block_streamer_service.rs +++ b/block-streamer/src/server/block_streamer_service.rs @@ -6,7 +6,7 @@ use tonic::{Request, Response, Status}; use crate::indexer_config::IndexerConfig; use crate::rules::types::ChainId; -use registry_types::{IndexerRule, IndexerRuleKind, MatchingRule}; +use registry_types::{IndexerRuleKind, MatchingRule, OldIndexerRule as IndexerRule}; use crate::block_stream; use crate::server::blockstreamer; diff --git a/coordinator/src/main.rs b/coordinator/src/main.rs index e308f1613..66322065b 100644 --- a/coordinator/src/main.rs +++ b/coordinator/src/main.rs @@ -255,7 +255,7 @@ mod tests { use mockall::predicate; use std::collections::HashMap; - use registry_types::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; + use registry_types::{IndexerRuleKind, MatchingRule, OldIndexerRule as IndexerRule, Status}; use crate::registry::IndexerConfig; diff --git a/coordinator/src/migration.rs b/coordinator/src/migration.rs index 38575b23b..514fbb596 100644 --- a/coordinator/src/migration.rs +++ b/coordinator/src/migration.rs @@ -281,7 +281,7 @@ mod tests { use std::collections::HashMap; use mockall::predicate; - use registry_types::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; + use registry_types::{IndexerRuleKind, MatchingRule, OldIndexerRule, Status}; use crate::registry::IndexerConfig; @@ -296,7 +296,7 @@ mod tests { function_name: "test".to_string(), code: String::new(), schema: Some(String::new()), - filter: IndexerRule { + filter: OldIndexerRule { id: None, name: None, indexer_rule_kind: IndexerRuleKind::Action, @@ -369,7 +369,7 @@ mod tests { function_name: "test".to_string(), code: String::new(), schema: Some(String::new()), - filter: IndexerRule { + filter: OldIndexerRule { id: None, name: None, indexer_rule_kind: IndexerRuleKind::Action, diff --git a/coordinator/src/registry.rs b/coordinator/src/registry.rs index db13c77af..b3ed7776e 100644 --- a/coordinator/src/registry.rs +++ b/coordinator/src/registry.rs @@ -8,7 +8,9 @@ use near_jsonrpc_client::JsonRpcClient; use near_jsonrpc_primitives::types::query::QueryResponseKind; use near_primitives::types::{AccountId, BlockReference, Finality, FunctionArgs}; use near_primitives::views::QueryRequest; -use registry_types::{AccountOrAllIndexers, IndexerRule}; +use registry_types::{ + OldAccountOrAllIndexers as AccountOrAllIndexers, OldIndexerRule as IndexerRule, +}; use crate::utils::exponential_retry; @@ -69,7 +71,7 @@ impl RegistryImpl { fn enrich_indexer_registry( &self, - registry: HashMap>, + registry: HashMap>, ) -> IndexerRegistry { registry .into_iter() diff --git a/registry/contract/src/lib.rs b/registry/contract/src/lib.rs index c9b000bff..f33835461 100644 --- a/registry/contract/src/lib.rs +++ b/registry/contract/src/lib.rs @@ -5,50 +5,45 @@ use near_sdk::store::UnorderedMap; use near_sdk::{env, log, near_bindgen, serde_json, AccountId, BorshStorageKey, CryptoHash}; use registry_types::{ - AccountOrAllIndexers, IndexerConfig, IndexerRule, IndexerRuleKind, MatchingRule, Status, + AccountIndexers, AllIndexers, IndexerConfig, IndexerRuleKind, MatchingRule, + OldAccountOrAllIndexers, OldIndexerConfig, OldIndexerRule, Rule, StartBlock, Status, }; type FunctionName = String; -// Define the contract structure -#[near_bindgen] + #[derive(BorshDeserialize, BorshSerialize, Debug)] -pub struct Contract { - registry: IndexersByAccount, +pub struct OldContract { + registry: OldIndexersByAccount, account_roles: Vec, } -pub type IndexersByAccount = UnorderedMap; +pub type OldIndexersByAccount = UnorderedMap; -pub type IndexerConfigByFunctionName = UnorderedMap; +pub type OldIndexerConfigByFunctionName = UnorderedMap; +// Define the contract structure +#[near_bindgen] #[derive(BorshDeserialize, BorshSerialize, Debug)] -pub struct OldState { - registry: OldIndexersByAccount, +pub struct Contract { + registry: IndexersByAccount, account_roles: Vec, } -#[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -#[serde(crate = "near_sdk::serde")] -pub struct OldIndexerConfig { - pub code: String, - pub start_block_height: Option, - pub schema: Option, - pub filter: IndexerRule, -} - -pub type OldIndexersByAccount = UnorderedMap; +type IndexersByAccount = UnorderedMap; -pub type OldIndexerConfigByFunctionName = UnorderedMap; +type IndexerConfigByFunctionName = UnorderedMap; // Migration types #[derive(BorshStorageKey, BorshSerialize)] -pub enum StorageKeys { +enum StorageKeys { Registry, // can be removed after migration Account(CryptoHash), // can be removed after migration RegistryV1, AccountV1(CryptoHash), RegistryV2, AccountV2(CryptoHash), + RegistryV3, + AccountV3(CryptoHash), } /// These roles are used to control access across the various contract methods. @@ -74,11 +69,10 @@ pub struct AccountRole { role: Role, } -// Define the default, which automatically initializes the contract impl Default for Contract { fn default() -> Self { Self { - registry: IndexersByAccount::new(StorageKeys::Registry), + registry: IndexersByAccount::new(StorageKeys::RegistryV3), account_roles: vec![ AccountRole { account_id: "morgs.near".parse().unwrap(), @@ -115,50 +109,29 @@ impl Contract { #[private] #[init(ignore_state)] pub fn migrate() -> Self { - let state: OldState = env::state_read().expect("failed to parse existing state"); + let state: OldContract = env::state_read().expect("failed to parse existing state"); - let mut registry = IndexersByAccount::new(StorageKeys::RegistryV2); + let mut registry = IndexersByAccount::new(StorageKeys::RegistryV3); for (account_id, indexers) in state.registry.iter() { let mut new_indexers: IndexerConfigByFunctionName = IndexerConfigByFunctionName::new( - StorageKeys::AccountV2(env::sha256_array(account_id.as_bytes())), + StorageKeys::AccountV3(env::sha256_array(account_id.as_bytes())), ); for (function_name, indexer_config) in indexers.iter() { - new_indexers.insert( - function_name.to_string(), - IndexerConfig { - updated_at_block_height: None, - created_at_block_height: env::block_height(), - schema: indexer_config.schema.clone(), - code: indexer_config.code.clone(), - start_block_height: indexer_config.start_block_height, - filter: indexer_config.filter.clone(), - }, - ); + new_indexers.insert(function_name.to_string(), indexer_config.clone().into()); } registry.insert(account_id.clone(), new_indexers); } - let account_roles: Vec<_> = Contract::default() - .account_roles - .into_iter() - .chain( - state - .account_roles - .into_iter() - .filter(|account_role| account_role.role == Role::User), - ) - .collect(); - Self { registry, - account_roles, + account_roles: state.account_roles, } } - pub fn near_social_indexer_rule() -> IndexerRule { + pub fn near_social_indexer_rule() -> OldIndexerRule { let contract = "social.near"; let method = "set"; let matching_rule = MatchingRule::ActionFunctionCall { @@ -166,7 +139,7 @@ impl Contract { function: method.to_string(), status: Status::Any, }; - IndexerRule { + OldIndexerRule { indexer_rule_kind: IndexerRuleKind::Action, matching_rule, id: None, @@ -179,7 +152,7 @@ impl Contract { &self, function_name: String, account_id: Option, - ) -> IndexerConfig { + ) -> OldIndexerConfig { let account_id = match account_id { Some(account_id) => account_id.parse::().unwrap_or_else(|_| { env::panic_str(&format!("Account ID {} is invalid", account_id)); @@ -191,7 +164,7 @@ impl Contract { env::panic_str(format!("Account {} has no registered functions", account_id).as_str()) }); - let indexer_config = account_indexers.get(&function_name).unwrap_or_else(|| { + let config = account_indexers.get(&function_name).unwrap_or_else(|| { env::panic_str( format!( "Function {} is not registered under account {}", @@ -201,7 +174,7 @@ impl Contract { ) }); - indexer_config.clone() + config.clone().into() } pub fn assert_roles(&self, permitted_roles: Vec) { @@ -279,6 +252,70 @@ impl Contract { }) } + pub fn register( + &mut self, + function_name: String, + code: String, + schema: String, + rule: Rule, + start_block: StartBlock, + ) { + let account_id = env::signer_account_id(); + + log!( + "Registering function {} for account {}", + &function_name, + &account_id + ); + + match &rule { + Rule::ActionAny { + affected_account_id, + .. + } + | Rule::ActionFunctionCall { + affected_account_id, + .. + } => { + if affected_account_id == "*" { + self.assert_roles(vec![Role::Owner]); + } + } + _ => {} + } + + let account_indexers = + self.registry + .entry(account_id.clone()) + .or_insert(IndexerConfigByFunctionName::new(StorageKeys::AccountV3( + env::sha256_array(account_id.as_bytes()), + ))); + + match account_indexers.entry(function_name) { + near_sdk::store::unordered_map::Entry::Occupied(mut entry) => { + let indexer = entry.get(); + entry.insert(IndexerConfig { + code, + schema, + rule, + start_block, + updated_at_block_height: Some(env::block_height()), + created_at_block_height: indexer.created_at_block_height, + }); + } + near_sdk::store::unordered_map::Entry::Vacant(entry) => { + entry.insert(IndexerConfig { + code, + schema, + rule, + start_block, + updated_at_block_height: None, + created_at_block_height: env::block_height(), + }); + } + } + } + // Public method - registers indexer code under then function_name pub fn register_indexer_function( &mut self, @@ -303,13 +340,29 @@ impl Contract { } }; - let filter: IndexerRule = match filter_json { + let filter: OldIndexerRule = match filter_json { Some(filter_json) => { - let filter_rule: IndexerRule = - serde_json::from_str(&filter_json).unwrap_or_else(|_| { - env::panic_str(&format!("Invalid filter JSON {}", filter_json)); + let filter_rule: OldIndexerRule = serde_json::from_str(&filter_json) + .unwrap_or_else(|e| { + env::panic_str(&format!("Invalid filter JSON {}", e)); }); + match &filter_rule.matching_rule { + MatchingRule::ActionAny { + affected_account_id, + .. + } + | MatchingRule::ActionFunctionCall { + affected_account_id, + .. + } => { + if affected_account_id == "*" { + self.assert_roles(vec![Role::Owner]); + } + } + _ => {} + } + filter_rule } None => Contract::near_social_indexer_rule(), @@ -324,18 +377,23 @@ impl Contract { let account_indexers = self.registry .entry(account_id.clone()) - .or_insert(IndexerConfigByFunctionName::new(StorageKeys::Account( + .or_insert(IndexerConfigByFunctionName::new(StorageKeys::AccountV3( env::sha256_array(account_id.as_bytes()), ))); + let start_block = match start_block_height { + Some(height) => StartBlock::Height(height), + None => StartBlock::Latest, + }; + match account_indexers.entry(function_name) { near_sdk::store::unordered_map::Entry::Occupied(mut entry) => { let indexer = entry.get(); entry.insert(IndexerConfig { code, - start_block_height, - schema, - filter, + start_block, + schema: schema.unwrap_or(String::new()), + rule: filter.matching_rule.into(), updated_at_block_height: Some(env::block_height()), created_at_block_height: indexer.created_at_block_height, }); @@ -343,9 +401,9 @@ impl Contract { near_sdk::store::unordered_map::Entry::Vacant(entry) => { entry.insert(IndexerConfig { code, - start_block_height, - schema, - filter, + start_block, + schema: schema.unwrap_or(String::new()), + rule: filter.matching_rule.into(), updated_at_block_height: None, created_at_block_height: env::block_height(), }); @@ -393,7 +451,7 @@ impl Contract { } } - pub fn list_indexer_functions(&self, account_id: Option) -> AccountOrAllIndexers { + pub fn list_indexer_functions(&self, account_id: Option) -> OldAccountOrAllIndexers { match account_id { Some(account_id) => { let account_id = account_id.parse::().unwrap_or_else(|_| { @@ -406,14 +464,16 @@ impl Contract { ) }); - AccountOrAllIndexers::Account( + OldAccountOrAllIndexers::Account( account_indexers .iter() - .map(|(function_name, config)| (function_name.clone(), config.clone())) + .map(|(function_name, config)| { + (function_name.clone(), config.clone().into()) + }) .collect(), ) } - None => AccountOrAllIndexers::All( + None => OldAccountOrAllIndexers::All( self.registry .iter() .map(|(account_id, account_indexers)| { @@ -422,7 +482,7 @@ impl Contract { account_indexers .iter() .map(|(function_name, config)| { - (function_name.clone(), config.clone()) + (function_name.clone(), config.clone().into()) }) .collect(), ) @@ -431,12 +491,34 @@ impl Contract { ), } } + + pub fn list_by_account(&self, account_id: AccountId) -> AccountIndexers { + self.registry + .get(&account_id) + .unwrap_or(&IndexerConfigByFunctionName::new(StorageKeys::AccountV3( + env::sha256_array(account_id.as_bytes()), + ))) + .iter() + .map(|(function_name, config)| (function_name.clone(), config.clone())) + .collect() + } + + pub fn list_all(&self) -> AllIndexers { + self.registry + .iter() + .map(|(account_id, account_indexers)| { + ( + account_id.clone(), + account_indexers + .iter() + .map(|(function_name, config)| (function_name.clone(), config.clone())) + .collect(), + ) + }) + .collect() + } } -/* - * The rest of this file holds the inline tests for the code above - * Learn more about Rust tests: https://doc.rust-lang.org/book/ch11-01-writing-tests.html - */ #[cfg(test)] mod tests { use super::*; @@ -445,9 +527,9 @@ mod tests { #[test] fn migrate() { - let mut registry = OldIndexersByAccount::new(StorageKeys::RegistryV1); + let mut registry = OldIndexersByAccount::new(StorageKeys::RegistryV2); let account_id = "morgs.near".parse::().unwrap(); - let mut functions = OldIndexerConfigByFunctionName::new(StorageKeys::AccountV1( + let mut functions = OldIndexerConfigByFunctionName::new(StorageKeys::AccountV2( env::sha256_array(account_id.as_bytes()), )); @@ -458,35 +540,34 @@ mod tests { start_block_height: None, schema: None, filter: Contract::near_social_indexer_rule(), + created_at_block_height: 10, + updated_at_block_height: None, }, ); functions.insert( "test2".to_string(), OldIndexerConfig { code: "return block2;".to_string(), - start_block_height: None, - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block_height: Some(100), + schema: Some(String::from("create table blah")), + filter: OldIndexerRule { + id: None, + name: None, + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: MatchingRule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, + }, + created_at_block_height: 10, + updated_at_block_height: Some(20), }, ); registry.insert(account_id.clone(), functions); - env::state_write(&OldState { + env::state_write(&OldContract { registry, - account_roles: vec![ - AccountRole { - account_id: account_id.clone(), - role: Role::Owner, - }, - AccountRole { - account_id: "should-be-removed.near".parse().unwrap(), - role: Role::Owner, - }, - AccountRole { - account_id: "bob.near".parse().unwrap(), - role: Role::User, - }, - ], + account_roles: Contract::default().account_roles, }); let contract = Contract::migrate(); @@ -500,11 +581,15 @@ mod tests { .unwrap(), &IndexerConfig { code: "return block;".to_string(), - start_block_height: None, - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionFunctionCall { + affected_account_id: String::from("social.near"), + status: Status::Any, + function: String::from("set") + }, updated_at_block_height: None, - created_at_block_height: env::block_height(), + created_at_block_height: 10, } ); assert_eq!( @@ -516,24 +601,17 @@ mod tests { .unwrap(), &IndexerConfig { code: "return block2;".to_string(), - start_block_height: None, - schema: None, - filter: Contract::near_social_indexer_rule(), - updated_at_block_height: None, - created_at_block_height: env::block_height(), + schema: String::from("create table blah"), + start_block: StartBlock::Height(100), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success + }, + updated_at_block_height: Some(20), + created_at_block_height: 10, } ); - assert_eq!( - contract.account_roles, - Contract::default() - .account_roles - .into_iter() - .chain(std::iter::once(AccountRole { - account_id: "bob.near".parse().unwrap(), - role: Role::User, - })) - .collect::>() - ); + assert_eq!(contract.account_roles, Contract::default().account_roles); } #[test] @@ -597,7 +675,7 @@ mod tests { assert!(contract .account_roles .iter() - .any(|account| account.account_id.to_string() == "alice.near")) + .any(|account| account.account_id == "alice.near")) } #[test] @@ -689,7 +767,7 @@ mod tests { assert!(!contract .account_roles .iter() - .any(|account| account.account_id.to_string() == "alice.near")) + .any(|account| account.account_id == "alice.near")) } #[test] @@ -747,7 +825,7 @@ mod tests { role: Role::User, }], }; - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: Some(43434343), schema: None, @@ -780,7 +858,7 @@ mod tests { role: Role::Owner, }], }; - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: Some(43434343), schema: None, @@ -895,7 +973,7 @@ mod tests { role: Role::User, }], }; - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: None, schema: None, @@ -914,13 +992,16 @@ mod tests { ); assert_eq!( - contract - .registry - .get(&"bob.near".parse::().unwrap()) - .unwrap() - .get("test") - .unwrap(), - &config + OldIndexerConfig::from( + contract + .registry + .get(&"bob.near".parse::().unwrap()) + .unwrap() + .get("test") + .unwrap() + .clone() + ), + config ); } @@ -963,11 +1044,11 @@ mod tests { role: Role::User, }], }; - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: None, schema: None, - filter: IndexerRule { + filter: OldIndexerRule { indexer_rule_kind: IndexerRuleKind::Action, matching_rule: MatchingRule::ActionFunctionCall { affected_account_id: "test".to_string(), @@ -991,13 +1072,16 @@ mod tests { ); assert_eq!( - contract - .registry - .get(&"bob.near".parse::().unwrap()) - .unwrap() - .get("test") - .unwrap(), - &config + OldIndexerConfig::from( + contract + .registry + .get(&"bob.near".parse::().unwrap()) + .unwrap() + .get("test") + .unwrap() + .clone() + ), + config ); } @@ -1010,11 +1094,11 @@ mod tests { role: Role::User, }], }; - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: None, schema: None, - filter: IndexerRule { + filter: OldIndexerRule { indexer_rule_kind: IndexerRuleKind::Action, matching_rule: MatchingRule::ActionAny { affected_account_id: "test".to_string(), @@ -1037,13 +1121,16 @@ mod tests { ); assert_eq!( - contract - .registry - .get(&"bob.near".parse::().unwrap()) - .unwrap() - .get("test") - .unwrap(), - &config + OldIndexerConfig::from( + contract + .registry + .get(&"bob.near".parse::().unwrap()) + .unwrap() + .get("test") + .unwrap() + .clone() + ), + config ); } @@ -1080,9 +1167,12 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, created_at_block_height: 100, }, @@ -1130,11 +1220,14 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); let mut registry = IndexersByAccount::new(StorageKeys::Registry); @@ -1146,7 +1239,7 @@ mod tests { role: Role::User, }], }; - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: None, schema: None, @@ -1174,6 +1267,45 @@ mod tests { ); } + #[test] + #[should_panic(expected = "Account bob.near does not have one of required roles [Owner]")] + fn prevents_non_owners_from_using_wildcard() { + let mut contract = Contract::default(); + contract.account_roles.push(AccountRole { + account_id: "bob.near".parse().unwrap(), + role: Role::User, + }); + + contract.register_indexer_function( + String::from("name"), + String::from("code"), + Some(0), + Some(String::from("schema")), + None, + Some(r#"{"indexer_rule_kind":"Action","matching_rule":{"rule":"ACTION_ANY","affected_account_id":"*","status":"SUCCESS"}}"#.to_string()), + ); + } + + #[test] + fn allows_owners_to_use_wildcard() { + let mut contract = Contract::default(); + contract.account_roles.push(AccountRole { + account_id: "bob.near".parse().unwrap(), + role: Role::Owner, + }); + + contract.register_indexer_function( + String::from("name"), + String::from("code"), + Some(0), + Some(String::from("schema")), + None, + Some(r#"{"indexer_rule_kind":"Action","matching_rule":{"rule":"ACTION_ANY","affected_account_id":"*","status":"SUCCESS"}}"#.to_string()), + ); + + assert_eq!(contract.registry.len(), 1); + } + #[test] fn users_can_remove_their_own_functions() { let account_id = "bob.near".parse::().unwrap(); @@ -1184,11 +1316,14 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); let mut registry = IndexersByAccount::new(StorageKeys::Registry); @@ -1219,11 +1354,14 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); let mut registry = IndexersByAccount::new(StorageKeys::Registry); @@ -1255,11 +1393,14 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); let mut registry = IndexersByAccount::new(StorageKeys::Registry); @@ -1285,11 +1426,14 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); let mut registry = IndexersByAccount::new(StorageKeys::Registry); @@ -1321,11 +1465,14 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); let mut registry = IndexersByAccount::new(StorageKeys::Registry); @@ -1348,22 +1495,28 @@ mod tests { "test".to_string(), IndexerConfig { code: "var x= 1;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); account_indexers.insert( "test2".to_string(), IndexerConfig { - code: "var x= 2;".to_string(), - start_block_height: Some(43434343), - schema: None, - filter: Contract::near_social_indexer_rule(), + code: "var x= 1;".to_string(), + start_block: StartBlock::Latest, + schema: String::new(), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Success, + }, updated_at_block_height: None, - created_at_block_height: 0, + created_at_block_height: 100, }, ); let mut registry = IndexersByAccount::new(StorageKeys::Registry); @@ -1407,7 +1560,7 @@ mod tests { #[test] fn read_indexer_function() { - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: None, schema: None, @@ -1415,11 +1568,12 @@ mod tests { updated_at_block_height: None, created_at_block_height: 0, }; + let account_id = "bob.near".parse::().unwrap(); let mut account_indexers = IndexerConfigByFunctionName::new(StorageKeys::Account( env::sha256_array(account_id.as_bytes()), )); - account_indexers.insert("test".to_string(), config.clone()); + account_indexers.insert("test".to_string(), config.clone().into()); let mut registry = IndexersByAccount::new(StorageKeys::Registry); registry.insert(account_id, account_indexers); let contract = Contract { @@ -1435,7 +1589,7 @@ mod tests { #[test] fn read_indexer_function_from_other_account() { - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: None, schema: None, @@ -1447,7 +1601,7 @@ mod tests { let mut account_indexers = IndexerConfigByFunctionName::new(StorageKeys::Account( env::sha256_array(account_id.as_bytes()), )); - account_indexers.insert("test".to_string(), config.clone()); + account_indexers.insert("test".to_string(), config.clone().into()); let mut registry = IndexersByAccount::new(StorageKeys::Registry); registry.insert(account_id, account_indexers); let contract = Contract { @@ -1471,7 +1625,7 @@ mod tests { #[test] fn list_indexer_functions() { - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: Some(43434343), schema: None, @@ -1483,7 +1637,7 @@ mod tests { let mut account_indexers = IndexerConfigByFunctionName::new(StorageKeys::Account( env::sha256_array(account_id.as_bytes()), )); - account_indexers.insert("test".to_string(), config.clone()); + account_indexers.insert("test".to_string(), config.clone().into()); let mut registry = IndexersByAccount::new(StorageKeys::Registry); registry.insert(account_id, account_indexers); let contract = Contract { @@ -1493,7 +1647,7 @@ mod tests { assert_eq!( contract.list_indexer_functions(None), - AccountOrAllIndexers::All(HashMap::from([( + OldAccountOrAllIndexers::All(HashMap::from([( "bob.near".parse().unwrap(), HashMap::from([("test".to_string(), config)]) )])) @@ -1502,7 +1656,7 @@ mod tests { #[test] fn list_account_indexer_functions() { - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: Some(43434343), schema: None, @@ -1514,7 +1668,7 @@ mod tests { let mut account_indexers = IndexerConfigByFunctionName::new(StorageKeys::Account( env::sha256_array(account_id.as_bytes()), )); - account_indexers.insert("test".to_string(), config.clone()); + account_indexers.insert("test".to_string(), config.clone().into()); let mut registry = IndexersByAccount::new(StorageKeys::Registry); registry.insert(account_id, account_indexers); let contract = Contract { @@ -1524,7 +1678,7 @@ mod tests { assert_eq!( contract.list_indexer_functions(Some("bob.near".to_string())), - AccountOrAllIndexers::Account(HashMap::from([("test".to_string(), config)])) + OldAccountOrAllIndexers::Account(HashMap::from([("test".to_string(), config)])) ); } @@ -1541,7 +1695,7 @@ mod tests { #[test] fn list_other_account_indexer_functions() { - let config = IndexerConfig { + let config = OldIndexerConfig { code: "var x= 1;".to_string(), start_block_height: Some(43434343), schema: None, @@ -1553,7 +1707,7 @@ mod tests { let mut account_indexers = IndexerConfigByFunctionName::new(StorageKeys::Account( env::sha256_array(account_id.as_bytes()), )); - account_indexers.insert("test".to_string(), config.clone()); + account_indexers.insert("test".to_string(), config.clone().into()); let mut registry = IndexersByAccount::new(StorageKeys::Registry); registry.insert(account_id, account_indexers); let contract = Contract { @@ -1563,7 +1717,99 @@ mod tests { assert_eq!( contract.list_indexer_functions(Some("alice.near".to_string())), - AccountOrAllIndexers::Account(HashMap::from([("test".to_string(), config)])) + OldAccountOrAllIndexers::Account(HashMap::from([("test".to_string(), config)])) + ); + } + + #[test] + fn list_all_indexers() { + let mut contract = Contract::default(); + + contract.register( + String::from("test"), + String::from("code"), + String::from("schema"), + Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Any, + }, + StartBlock::Latest, + ); + + assert_eq!( + contract.list_all(), + HashMap::from([( + "bob.near".parse::().unwrap(), + HashMap::from([( + String::from("test"), + IndexerConfig { + code: String::from("code"), + schema: String::from("schema"), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Any, + }, + start_block: StartBlock::Latest, + updated_at_block_height: None, + created_at_block_height: env::block_height(), + } + )]) + )]) + ); + } + + #[test] + fn list_empty_account_indexers() { + let mut contract = Contract::default(); + + contract.register( + String::from("test"), + String::from("code"), + String::from("schema"), + Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Any, + }, + StartBlock::Latest, + ); + + assert_eq!( + contract.list_by_account("morgs.near".parse().unwrap()), + HashMap::new() + ); + } + + #[test] + fn list_account_indexers() { + let mut contract = Contract::default(); + + contract.register( + String::from("test"), + String::from("code"), + String::from("schema"), + Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Any, + }, + StartBlock::Latest, + ); + + assert_eq!( + contract.list_by_account(env::signer_account_id()), + HashMap::from([( + String::from("test"), + IndexerConfig { + code: String::from("code"), + schema: String::from("schema"), + rule: Rule::ActionAny { + affected_account_id: String::from("social.near"), + status: Status::Any, + }, + start_block: StartBlock::Latest, + updated_at_block_height: None, + created_at_block_height: env::block_height(), + } + )]) ); } } diff --git a/registry/types/src/lib.rs b/registry/types/src/lib.rs index ddbd16bd8..ee9d1deaa 100644 --- a/registry/types/src/lib.rs +++ b/registry/types/src/lib.rs @@ -16,14 +16,6 @@ use serde::{Deserialize, Serialize}; type FunctionName = String; -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub enum Status { - Any, - Success, - Fail, -} - #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] #[serde(tag = "rule", rename_all = "SCREAMING_SNAKE_CASE")] pub enum MatchingRule { @@ -44,6 +36,40 @@ pub enum MatchingRule { }, } +impl From for MatchingRule { + fn from(value: Rule) -> Self { + match value { + Rule::ActionAny { + affected_account_id, + status, + } => MatchingRule::ActionAny { + affected_account_id, + status, + }, + Rule::Event { + contract_account_id, + standard, + version, + event, + } => MatchingRule::Event { + contract_account_id, + standard, + version, + event, + }, + Rule::ActionFunctionCall { + affected_account_id, + status, + function, + } => MatchingRule::ActionFunctionCall { + affected_account_id, + status, + function, + }, + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] pub enum IndexerRuleKind { Action, @@ -53,25 +79,160 @@ pub enum IndexerRuleKind { } #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] -pub struct IndexerRule { +pub struct OldIndexerRule { pub indexer_rule_kind: IndexerRuleKind, pub matching_rule: MatchingRule, + // These are not set, and not used anywhere pub id: Option, pub name: Option, } #[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -pub struct IndexerConfig { +pub struct OldIndexerConfig { pub code: String, pub start_block_height: Option, pub schema: Option, - pub filter: IndexerRule, + pub filter: OldIndexerRule, pub updated_at_block_height: Option, pub created_at_block_height: u64, } +impl From for OldIndexerConfig { + fn from(config: IndexerConfig) -> Self { + let start_block_height = match config.start_block { + StartBlock::Latest => None, + StartBlock::Continue => None, + StartBlock::Height(height) => Some(height), + }; + + let schema = if config.schema.is_empty() { + None + } else { + Some(config.schema) + }; + + OldIndexerConfig { + start_block_height, + schema, + code: config.code, + filter: OldIndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: config.rule.into(), + id: None, + name: None, + }, + created_at_block_height: config.created_at_block_height, + updated_at_block_height: config.updated_at_block_height, + } + } +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -pub enum AccountOrAllIndexers { - All(HashMap>), - Account(HashMap), +pub enum OldAccountOrAllIndexers { + All(HashMap>), + Account(HashMap), +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Status { + Any, + Success, + Fail, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Rule { + ActionAny { + affected_account_id: String, + status: Status, + }, + ActionFunctionCall { + affected_account_id: String, + status: Status, + function: String, + }, + Event { + contract_account_id: String, + standard: String, + version: String, + event: String, + }, } + +impl From for Rule { + fn from(value: MatchingRule) -> Self { + match value { + MatchingRule::ActionAny { + affected_account_id, + status, + } => Rule::ActionAny { + affected_account_id, + status, + }, + MatchingRule::Event { + contract_account_id, + standard, + version, + event, + } => Rule::Event { + contract_account_id, + standard, + version, + event, + }, + MatchingRule::ActionFunctionCall { + affected_account_id, + status, + function, + } => Rule::ActionFunctionCall { + affected_account_id, + status, + function, + }, + } + } +} + +#[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum StartBlock { + /// Specifies the particular block height from which to start indexing from. + Height(u64), + /// Starts indexing from the most recently finalized block. + Latest, + /// Resumes indexing from the block immediately following the last one successfully indexed + /// prior to update. + Continue, +} + +#[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct IndexerConfig { + pub code: String, + pub start_block: StartBlock, + pub schema: String, + pub rule: Rule, + pub updated_at_block_height: Option, + pub created_at_block_height: u64, +} + +impl From for IndexerConfig { + fn from(config: OldIndexerConfig) -> Self { + Self { + start_block: match config.start_block_height { + Some(height) => StartBlock::Height(height), + None => StartBlock::Latest, + }, + schema: config.schema.unwrap_or(String::new()), + code: config.code, + rule: config.filter.matching_rule.into(), + created_at_block_height: config.created_at_block_height, + updated_at_block_height: config.updated_at_block_height, + } + } +} + +pub type AccountIndexers = HashMap; + +pub type AllIndexers = HashMap;