Skip to content

Commit

Permalink
refactor: Extract registry types in to own crate (#453)
Browse files Browse the repository at this point in the history
This PR extract the types exposed by the registry contract in to its own
reusable crate: `registry_types`. This was done to avoid the duplication
across our different service which; fetch the JSON, and then construct
the Rust types.

## `registry_types` crate 
The registry types were already partially shared between Coordinator and
the registry, but I've expanded on this to include all types rather than
just a subset. I've created a new crate, rather than using the existing
`./indexer/indexer_rule_type` crate as some packages needed to be
upgraded, which caused conflicts across the `indexer/` cargo workspace -
which seemed unnecessary to fix given that it is being deprecated.

## Registry contract updates
`near-sdk` version `4.1.1` is no longer compilable because it depends on
a non-existent crate, hence the upgrade to `5.0.0-alpha.1`. Since I was
already making changes here, I decided to remove the migration related
code as it is no longer needed.

## Consuming `registry_types`
With the shared types in place, I've updated `block-streamer/` to use
them, removing the existing types which it duplicates. I also went ahead
and removed all types which were unused (left over from Alertexer).

`indexer_rule_type` depends on `near-sdk` version `4.1.1` (the broken
one), and since it's no longer shared, I removed that dependency so that
it can compile again.

Relates to #421
  • Loading branch information
morgsmccauley committed Dec 15, 2023
1 parent 663d4bc commit af38be3
Show file tree
Hide file tree
Showing 26 changed files with 7,548 additions and 1,978 deletions.
104 changes: 96 additions & 8 deletions block-streamer/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions block-streamer/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ tokio-stream = "0.1.14"
tonic = "0.10.2"
wildmatch = "2.1.1"

registry-types = { path = "../registry/types" }

near-lake-framework = "0.7.4"

[build-dependencies]
Expand Down
12 changes: 6 additions & 6 deletions block-streamer/src/block_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ use near_lake_framework::near_indexer_primitives;
use tokio::task::JoinHandle;

use crate::indexer_config::IndexerConfig;
use crate::rules::types::indexer_rule_match::ChainId;
use crate::rules::MatchingRule;
use crate::rules::types::ChainId;
use registry_types::MatchingRule;

/// The number of blocks to prefetch within `near-lake-framework`. The internal default is 100, but
/// we need this configurable for testing purposes.
Expand Down Expand Up @@ -251,11 +251,11 @@ mod tests {
)
.unwrap(),
function_name: "test".to_string(),
indexer_rule: crate::rules::IndexerRule {
indexer_rule_kind: crate::rules::IndexerRuleKind::Action,
matching_rule: crate::rules::MatchingRule::ActionAny {
indexer_rule: registry_types::IndexerRule {
indexer_rule_kind: registry_types::IndexerRuleKind::Action,
matching_rule: registry_types::MatchingRule::ActionAny {
affected_account_id: "queryapi.dataplatform.near".to_string(),
status: crate::rules::Status::Success,
status: registry_types::Status::Success,
},
name: None,
id: None,
Expand Down
3 changes: 2 additions & 1 deletion block-streamer/src/delta_lake_client.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
use crate::rules::types::indexer_rule_match::ChainId;
use anyhow::Context;
use chrono::TimeZone;
use futures::future::try_join_all;
use near_lake_framework::near_indexer_primitives;

use crate::rules::types::ChainId;

const DELTA_LAKE_BUCKET: &str = "near-delta-lake";
const MAX_S3_RETRY_COUNT: u8 = 20;
const INDEXED_ACTIONS_PREFIX: &str = "silver/accounts/action_receipt_actions/metadata";
Expand Down
11 changes: 2 additions & 9 deletions block-streamer/src/indexer_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,9 @@ use near_lake_framework::near_indexer_primitives::types::AccountId;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};

use crate::rules::IndexerRule;
use registry_types::IndexerRule;

#[derive(
borsh::BorshSerialize,
borsh::BorshDeserialize,
serde::Serialize,
serde::Deserialize,
Clone,
Debug,
)]
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
pub struct IndexerConfig {
pub account_id: AccountId,
pub function_name: String,
Expand Down
4 changes: 2 additions & 2 deletions block-streamer/src/rules/matcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ use near_lake_framework::near_indexer_primitives::{
views::{ActionView, ExecutionStatusView, ReceiptEnumView},
IndexerExecutionOutcomeWithReceipt,
};
use registry_types::{MatchingRule, Status};

use crate::rules::types::events::Event;
use crate::rules::{MatchingRule, Status};
use crate::rules::types::Event;

pub fn matches(
matching_rule: &MatchingRule,
Expand Down
57 changes: 2 additions & 55 deletions block-streamer/src/rules/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,62 +3,9 @@ pub mod outcomes_reducer;
pub mod types;

use near_lake_framework::near_indexer_primitives::StreamerMessage;
use types::indexer_rule_match::{ChainId, IndexerRuleMatch};
use registry_types::{IndexerRule, MatchingRule};

#[cfg(not(feature = "near-sdk"))]
use borsh::{self, BorshDeserialize, BorshSerialize};
#[cfg(not(feature = "near-sdk"))]
use serde::{Deserialize, Serialize};

#[cfg(feature = "near-sdk")]
use near_sdk::borsh::{self, BorshDeserialize, BorshSerialize};
#[cfg(feature = "near-sdk")]
use near_sdk::serde::{Deserialize, Serialize};

#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)]
pub struct IndexerRule {
pub indexer_rule_kind: IndexerRuleKind,
pub matching_rule: MatchingRule,
pub id: Option<u32>,
pub name: Option<String>,
}

#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)]
pub enum IndexerRuleKind {
Action,
Event,
AnyBlock,
Shard,
}
// future: ComposedRuleKind for multiple actions or events

#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum Status {
Any,
Success,
Fail,
}

#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)]
#[serde(tag = "rule", rename_all = "SCREAMING_SNAKE_CASE")]
pub enum MatchingRule {
ActionAny {
affected_account_id: String,
status: Status,
},
ActionFunctionCall {
affected_account_id: String,
status: Status,
function: String,
},
Event {
contract_account_id: String,
standard: String,
version: String,
event: String,
},
}
use types::{ChainId, IndexerRuleMatch};

pub fn reduce_indexer_rule_matches(
indexer_rule: &IndexerRule,
Expand Down
9 changes: 5 additions & 4 deletions block-streamer/src/rules/outcomes_reducer.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::rules::matcher;
use crate::rules::types::events::Event;
use crate::rules::types::indexer_rule_match::{ChainId, IndexerRuleMatch, IndexerRuleMatchPayload};
use crate::rules::types::Event;
use crate::rules::types::{ChainId, IndexerRuleMatch, IndexerRuleMatchPayload};
use crate::rules::{IndexerRule, MatchingRule};
use near_lake_framework::near_indexer_primitives::{
IndexerExecutionOutcomeWithReceipt, StreamerMessage,
Expand Down Expand Up @@ -115,9 +115,10 @@ fn build_indexer_rule_match_payload(

#[cfg(test)]
mod tests {
use registry_types::{IndexerRule, IndexerRuleKind, MatchingRule, Status};

use crate::rules::outcomes_reducer::reduce_indexer_rule_matches_from_outcomes;
use crate::rules::types::indexer_rule_match::{ChainId, IndexerRuleMatch};
use crate::rules::{IndexerRule, IndexerRuleKind, MatchingRule, Status};
use crate::rules::types::{ChainId, IndexerRuleMatch};

#[tokio::test]
async fn match_wildcard_no_match() {
Expand Down
Loading

0 comments on commit af38be3

Please sign in to comment.