Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improvements to integration test handling of execution options. #136

Merged
merged 2 commits into from
Jul 20, 2023
Merged

Improvements to integration test handling of execution options. #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
621aef5
Improvements to integration test handling of execution options.
gabehamilton Jul 18, 2023
163caeb
cargo fmt
gabehamilton Jul 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions indexer/queryapi_coordinator/src/historical_block_processing.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@ pub fn spawn_historical_message_thread(
tokio::spawn(process_historical_messages(
block_height,
new_indexer_function_copy,
Opts::parse(),
))
})
}

pub(crate) async fn process_historical_messages(
block_height: BlockHeight,
indexer_function: IndexerFunction,
opts: Opts,
) -> i64 {
let start_block = indexer_function.start_block_height.unwrap();
let block_difference: i64 = (block_height - start_block) as i64;
Expand All @@ -58,8 +60,6 @@ pub(crate) async fn process_historical_messages(
indexer_function.function_name
);

let opts = Opts::parse();

let chain_id = opts.chain_id().clone();
let aws_region = opts.aws_queue_region.clone();
let queue_client = queue::queue_client(aws_region, opts.queue_credentials());
Expand Down
279 changes: 153 additions & 126 deletions indexer/queryapi_coordinator/src/historical_block_processing_integration_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,140 +1,167 @@
use crate::historical_block_processing::filter_matching_blocks_from_index_files;
use crate::indexer_types::IndexerFunction;
use crate::opts::{Opts, Parser};
use crate::{historical_block_processing, opts};
use aws_types::SdkConfig;
use chrono::{DateTime, NaiveDate, Utc};
use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status};
use near_lake_framework::near_indexer_primitives::types::BlockHeight;
use std::ops::Range;
#[cfg(test)]
mod tests {
use crate::historical_block_processing::filter_matching_blocks_from_index_files;
use crate::indexer_types::IndexerFunction;
use crate::opts::{ChainId, Opts, StartOptions};
use crate::{historical_block_processing, opts};
use aws_types::SdkConfig;
use chrono::{DateTime, NaiveDate, Utc};
use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status};
use near_lake_framework::near_indexer_primitives::types::BlockHeight;
use std::env;
use std::ops::Range;

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_indexing_metadata_file -- mainnet from-latest;
#[tokio::test]
async fn test_indexing_metadata_file() {
let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
impl Opts {
pub fn test_opts_with_aws() -> Self {
dotenv::dotenv().ok();
let lake_aws_access_key = env::var("LAKE_AWS_ACCESS_KEY").unwrap();
let lake_aws_secret_access_key = env::var("LAKE_AWS_SECRET_ACCESS_KEY").unwrap();
Opts {
redis_connection_string: "".to_string(),
lake_aws_access_key,
lake_aws_secret_access_key,
queue_aws_access_key: "".to_string(),
queue_aws_secret_access_key: "".to_string(),
aws_queue_region: "".to_string(),
queue_url: "".to_string(),
start_from_block_queue_url: "".to_string(),
registry_contract_id: "".to_string(),
port: 0,
chain_id: ChainId::Mainnet(StartOptions::FromLatest),
}
}
}

let last_indexed_block =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
let a: Range<u64> = 90000000..9000000000; // valid for the next 300 years
assert!(a.contains(&last_indexed_block));
}
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_indexing_metadata_file -- mainnet from-latest;
#[tokio::test]
async fn test_indexing_metadata_file() {
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_process_historical_messages -- mainnet from-latest;
#[tokio::test]
async fn test_process_historical_messages() {
opts::init_tracing();
let last_indexed_block =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
let a: Range<u64> = 90000000..9000000000; // valid for the next 300 years
assert!(a.contains(&last_indexed_block));
}

let contract = "queryapi.dataplatform.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
let indexer_function = IndexerFunction {
account_id: "buildnear.testnet".to_string().parse().unwrap(),
function_name: "index_stuff".to_string(),
code: "".to_string(),
start_block_height: Some(85376002),
schema: None,
provisioned: false,
indexer_rule: filter_rule,
};
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_process_historical_messages -- mainnet from-latest;
#[tokio::test]
async fn test_process_historical_messages() {
opts::init_tracing();

let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
let fake_block_height =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
historical_block_processing::process_historical_messages(
fake_block_height + 1,
indexer_function,
)
.await;
}
let contract = "queryapi.dataplatform.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
let indexer_function = IndexerFunction {
account_id: "buildnear.testnet".to_string().parse().unwrap(),
function_name: "index_stuff".to_string(),
code: "".to_string(),
start_block_height: Some(85376002),
schema: None,
provisioned: false,
indexer_rule: filter_rule,
};

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_wildcard_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_wildcard_blocks_from_index_files() {
let contract = "*.keypom.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
let fake_block_height =
historical_block_processing::last_indexed_block_from_metadata(aws_config)
.await
.unwrap();
historical_block_processing::process_historical_messages(
fake_block_height + 1,
indexer_function,
opts,
)
.await;
}

let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_wildcard_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_wildcard_blocks_from_index_files() {
let contract = "*.keypom.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};

let start_block_height = 75472603;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2022, 10, 03)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();

// // remove any blocks from after when the test was written -- not working, due to new contracts?
// let fixed_blocks: Vec<BlockHeight> = blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); // 95175853u64 95242647u64
assert!(blocks.len() > 21830); // 22913 raw, deduped to 21830
}
let start_block_height = 77016214;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2022, 10, 03)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;

/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_blocks_from_index_files() {
let contract = "*.agency.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};
// // remove any blocks from after when the test was written -- not working, due to new contracts?
// let fixed_blocks: Vec<BlockHeight> = blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); // 95175853u64 95242647u64
assert!(blocks.len() >= 71899);
}

let opts = Opts::parse();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();
/// Parses env vars from .env, Run with
/// cargo test historical_block_processing_integration_tests::test_filter_matching_blocks_from_index_files -- mainnet from-latest;
#[tokio::test]
async fn test_filter_matching_blocks_from_index_files() {
let contract = "*.agency.near";
let matching_rule = MatchingRule::ActionAny {
affected_account_id: contract.to_string(),
status: Status::Any,
};
let filter_rule = IndexerRule {
indexer_rule_kind: IndexerRuleKind::Action,
matching_rule,
id: None,
name: None,
};

let start_block_height = 45894620;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2021, 08, 01)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;
let opts = Opts::test_opts_with_aws();
let aws_config: &SdkConfig = &opts.lake_aws_sdk_config();

let start_block_height = 45894620;
let naivedatetime_utc = NaiveDate::from_ymd_opt(2021, 08, 01)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let datetime_utc = DateTime::<Utc>::from_utc(naivedatetime_utc, Utc);
let blocks = filter_matching_blocks_from_index_files(
start_block_height,
&filter_rule,
aws_config,
datetime_utc,
)
.await;

// remove any blocks from after when the test was written
let fixed_blocks: Vec<BlockHeight> = blocks.into_iter().filter(|&b| b <= 95175853u64).collect();
assert_eq!(fixed_blocks.len(), 6); // hackathon.agency.near = 45894627,45898423, hacker.agency.near = 45897358, hack.agency.near = 45894872,45895120,45896237
// remove any blocks from after when the test was written
let fixed_blocks: Vec<BlockHeight> =
blocks.into_iter().filter(|&b| b <= 95175853u64).collect();
assert_eq!(fixed_blocks.len(), 6); // hackathon.agency.near = 45894627,45898423, hacker.agency.near = 45897358, hack.agency.near = 45894872,45895120,45896237
}
}