Skip to content

Commit

Permalink
Merge pull request #136 from near/dplt-1069_integration_test_improvem…
Browse files Browse the repository at this point in the history 
…ents

Improvements to integration test handling of execution options.
  • Loading branch information
@gabehamilton
gabehamilton authored Jul 20, 2023
2 parents d700a85 + 163caeb commit 316f9db
Show file tree
Hide file tree
Showing 2 changed files with 155 additions and 128 deletions.
4 changes: 2 additions & 2 deletions indexer/queryapi_coordinator/src/historical_block_processing.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@ pub fn spawn_historical_message_thread(
tokio::spawn(process_historical_messages(
block_height,
new_indexer_function_copy,
Opts::parse(),
))
})
}

pub(crate) async fn process_historical_messages(
block_height: BlockHeight,
indexer_function: IndexerFunction,
opts: Opts,
) -> i64 {
let start_block = indexer_function.start_block_height.unwrap();
let block_difference: i64 = (block_height - start_block) as i64;
Expand All @@ -58,8 +60,6 @@ pub(crate) async fn process_historical_messages(
indexer_function.function_name
);

let opts = Opts::parse();

let chain_id = opts.chain_id().clone();
let aws_region = opts.aws_queue_region.clone();
let queue_client = queue::queue_client(aws_region, opts.queue_credentials());
Expand Down
279 changes: 153 additions & 126 deletions indexer/queryapi_coordinator/src/historical_block_processing_integration_tests.rs
View file
Original file line number Diff line number Diff line change
@@ -1,140 +1,167 @@
use crate::historical_block_processing::filter_matching_blocks_from_index_files;
use crate::indexer_types::IndexerFunction;
use crate::opts::{Opts, Parser};
use crate::{historical_block_processing, opts};
use aws_types::SdkConfig;
use chrono::{DateTime, NaiveDate, Utc};
use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status};
use near_lake_framework::near_indexer_primitives::types::BlockHeight;
use std::ops::Range;
#[cfg(test)]
mod tests {
use crate::historical_block_processing::filter_matching_blocks_from_index_files;
use crate::indexer_types::IndexerFunction;
use crate::opts::{ChainId, Opts, StartOptions};
use crate::{historical_block_processing, opts};
use aws_types::SdkConfig;
use chrono::{DateTime, NaiveDate, Utc};
use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status};
use near_lake_framework::near_indexer_primitives::types::BlockHeight;
use std::env;
use std::ops::Range;

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_indexing_metadata_file -- mainnet from-latest;
#[tokio::test]
async fn test_indexing_metadata_file() {
let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
impl Opts {
pub fn test_opts_with_aws() -> Self {
dotenv::dotenv().ok();
let lake_aws_access_key = env::var("LAKE_AWS_ACCESS_KEY").unwrap();
let lake_aws_secret_access_key = env::var("LAKE_AWS_SECRET_ACCESS_KEY").unwrap();
Opts {
redis_connection_string: "".to_string(),
lake_aws_access_key,
lake_aws_secret_access_key,
queue_aws_access_key: "".to_string(),
queue_aws_secret_access_key: "".to_string(),
aws_queue_region: "".to_string(),
queue_url: "".to_string(),
start_from_block_queue_url: "".to_string(),
registry_contract_id: "".to_string(),
port: 0,
chain_id: ChainId::Mainnet(StartOptions::FromLatest),
}
}
}

let last_indexed_block =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
let a: Range<u64> = 90000000..9000000000; // valid for the next 300 years
assert!(a.contains(&last_indexed_block));
}
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_indexing_metadata_file -- mainnet from-latest;
#[tokio::test]
async fn test_indexing_metadata_file() {
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_process_historical_messages -- mainnet from-latest;
#[tokio::test]
async fn test_process_historical_messages() {
opts::init_tracing();
let last_indexed_block =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
let a: Range<u64> = 90000000..9000000000; // valid for the next 300 years
assert!(a.contains(&last_indexed_block));
}

let contract = "queryapi.dataplatform.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
let indexer_function = IndexerFunction {
account_id: "buildnear.testnet".to_string().parse().unwrap(),
function_name: "index_stuff".to_string(),
code: "".to_string(),
start_block_height: Some(85376002),
schema: None,
provisioned: false,
indexer_rule: filter_rule,
};
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_process_historical_messages -- mainnet from-latest;
#[tokio::test]
async fn test_process_historical_messages() {
opts::init_tracing();

let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
let fake_block_height =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
historical_block_processing::process_historical_messages(
fake_block_height + 1,
indexer_function,
)
.await;
}
let contract = "queryapi.dataplatform.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
let indexer_function = IndexerFunction {
account_id: "buildnear.testnet".to_string().parse().unwrap(),
function_name: "index_stuff".to_string(),
code: "".to_string(),
start_block_height: Some(85376002),
schema: None,
provisioned: false,
indexer_rule: filter_rule,
};

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_wildcard_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_wildcard_blocks_from_index_files() {
let contract = "*.keypom.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
let fake_block_height =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
historical_block_processing::process_historical_messages(
fake_block_height + 1,
indexer_function,
opts,
)
.await;
}

let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_wildcard_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_wildcard_blocks_from_index_files() {
let contract = "*.keypom.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};

let start_block_height = 75472603;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2022, 10, 03)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();

// // remove any blocks from after when the test was written -- not working, due to new contracts?
// let fixed_blocks: Vec<BlockHeight> = blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); // 95175853u64 95242647u64
assert!(blocks.len() > 21830); // 22913 raw, deduped to 21830
}
let start_block_height = 77016214;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2022, 10, 03)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_blocks_from_index_files() {
let contract = "*.agency.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
// // remove any blocks from after when the test was written -- not working, due to new contracts?
// let fixed_blocks: Vec<BlockHeight> = blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); // 95175853u64 95242647u64
assert!(blocks.len() >= 71899);
}

let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_blocks_from_index_files() {
let contract = "*.agency.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};

let start_block_height = 45894620;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2021, 08, 01)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();

let start_block_height = 45894620;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2021, 08, 01)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;

// remove any blocks from after when the test was written
let fixed_blocks: Vec<BlockHeight> = blocks.into_iter().filter(|&b| b <= 95175853u64).collect();
assert_eq!(fixed_blocks.len(), 6); // hackathon.agency.near = 45894627,45898423, hacker.agency.near = 45897358, hack.agency.near = 45894872,45895120,45896237
// remove any blocks from after when the test was written
let fixed_blocks: Vec<BlockHeight> =
blocks.into_iter().filter(|&b| b <= 95175853u64).collect();
assert_eq!(fixed_blocks.len(), 6); // hackathon.agency.near = 45894627,45898423, hacker.agency.near = 45897358, hack.agency.near = 45894872,45895120,45896237
}
}

0 comments on commit 316f9db

Please sign in to comment.