DPLT-980 Tests for wildcard matching

.gitignore

@@ -2,3 +2,4 @@
 .env*
 redis/
 *.log
+/indexer/blocks/

block-server/handler.js

@@ -13,7 +13,9 @@ module.exports.block = async (event) => {
   try {
     // parse request params
     const { block_height } = event.pathParameters;
-    const block = await fetchStreamerMessage(block_height);
+    const options = event.queryStringParameters || {};
+    options.snake_case = options.snake_case === 'true';
+    const block = await fetchStreamerMessage(block_height, options);
     return {
       statusCode: 200,
       headers,
@@ -32,10 +34,10 @@ const normalizeBlockHeight = function(block_height) {
     return block_height.toString().padStart(12, '0');
 }
 
-const fetchStreamerMessage = async function(block_height) {
-    const blockPromise = fetchBlockPromise(block_height);
+const fetchStreamerMessage = async function(block_height, options) {
+    const blockPromise = fetchBlockPromise(block_height, options);
     // hardcoding 4 shards to test performance
-    const shardsPromises = await fetchShardsPromises(block_height, 4); // block.chunks.length)
+    const shardsPromises = await fetchShardsPromises(block_height, 4, options); // block.chunks.length)
 
     const results = await Promise.all([blockPromise, ...shardsPromises]);
     const block = results.shift();
@@ -46,30 +48,37 @@ const fetchStreamerMessage = async function(block_height) {
     };
 }
 
-const fetchShardsPromises = async function(block_height, number_of_shards) {
+const fetchShardsPromises = async function(block_height, number_of_shards, options) {
     return ([...Array(number_of_shards).keys()].map((shard_id) =>
-        fetchShardPromise(block_height, shard_id)));
+        fetchShardPromise(block_height, shard_id, options)));
 }
 
-const fetchShardPromise = function(block_height, shard_id) {
+const fetchShardPromise = function(block_height, shard_id, options) {
     const params = {
         Bucket: `near-lake-data-${NETWORK}`,
         Key: `${normalizeBlockHeight(block_height)}/shard_${shard_id}.json`,
     };
     return S3.getObject(params).promise().then((response) => {
-        return JSON.parse(response.Body.toString(), (key, value) => renameUnderscoreFieldsToCamelCase(value));
+        return JSON.parse(response.Body.toString(), (key, value) => {
+            if(options.snake_case) return value
+            return renameUnderscoreFieldsToCamelCase(value)
+        });
     });
 }
 
-const fetchBlockPromise = function(block_height) {
+const fetchBlockPromise = function(block_height, options) {
     const file = 'block.json';
     const folder = normalizeBlockHeight(block_height);
     const params = {
         Bucket: 'near-lake-data-' + NETWORK,
         Key: `${folder}/${file}`,
     };
     return S3.getObject(params).promise().then((response) => {
-        const block = JSON.parse(response.Body.toString(), (key, value) => renameUnderscoreFieldsToCamelCase(value));
+        const block = JSON.parse(response.Body.toString(), (key, value) => {
+            if(options.snake_case) return value
+            return renameUnderscoreFieldsToCamelCase(value)
+        });
+
         return block;
     });
 }

indexer/README.md

@@ -17,13 +17,20 @@ This project is using `workspace` feature of Cargo.
 
 ### Crates
 
-- [`alert-rules`](./alert-rules) crate provides the `AlertRule` type for usage in other crates
-- [`shared`](./shared) crate holds the common `clap` structs for every indexer in the workspace. Also, it includes shared types and utils.
-- [`storage`](./storage) crate provides the functions to work with Redis that are common for all indexers in the workspace
+- [`indexer_rule_type`](./indexer_rule_type) provides the IndexerRule type to this app and the registry contract.
+- [`indexer_rules_engine`](./indexer_rules_engine) contains logic for matching IndexerRules against StreamerMessages
+- [`storage`](./storage) crate provides the functions to work with Redis
 
 ### Indexers
 
-- [`queryapi_coordinator`](./queryapi_coordinator) an indexer to watch for `AlertRules` and index changes to the QueryApi registry contract.
+- [`queryapi_coordinator`](./queryapi_coordinator) an indexer to index changes to the QueryApi registry contract and
+  to watch for `IndexerRules` associated with the IndexerFunctions in the registry.
+
+### Tests
+Some tests require blocks with matching data. To download the test block, run 
+`./download_test_blocks.sh 93085141`. Some other useful blocks are 80854399 92476362 93085141 93659695.
+
+To log a message instead of sending SQS messages set your `QUEUE_URL` to `MOCK` in your `.env` file.
 
 ## Design concept
 

indexer/download_test_blocks.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+mkdir -p ./blocks
+
+# Iterate over all script arguments
+for block_id in "$@"
+do
+  curl -o "./blocks/${block_id}.json" "https://70jshyr5cb.execute-api.eu-central-1.amazonaws.com/block/${block_id}?snake_case=true"
+done

indexer/indexer_rules_engine/Cargo.toml

@@ -15,3 +15,6 @@ wildmatch = "2.1.1"
 
 near-lake-framework = "0.7.1"
 indexer_rule_type = { path = "../indexer_rule_type" }
+
+[dev-dependencies]
+tokio = { version = "1.0.1", features = ["full"] }

indexer/indexer_rules_engine/src/matcher.rs

@@ -114,9 +114,16 @@ fn match_account(
     account_id: &str,
     outcome_with_receipt: &IndexerExecutionOutcomeWithReceipt,
 ) -> bool {
-    wildmatch::WildMatch::new(account_id).matches(&outcome_with_receipt.receipt.receiver_id)
-        || wildmatch::WildMatch::new(account_id)
-            .matches(&outcome_with_receipt.receipt.predecessor_id)
+    match account_id {
+        x if x.contains(",") => x
+            .split(",")
+            .any(|sub_account_id| match_account(sub_account_id.trim(), outcome_with_receipt)),
+        _ => {
+            wildmatch::WildMatch::new(account_id).matches(&outcome_with_receipt.receipt.receiver_id)
+                || wildmatch::WildMatch::new(account_id)
+                    .matches(&outcome_with_receipt.receipt.predecessor_id)
+        }
+    }
 }
 
 fn match_status(status: &Status, execution_outcome_status: &ExecutionStatusView) -> bool {

indexer/indexer_rules_engine/src/outcomes_reducer.rs

@@ -115,3 +115,160 @@ fn build_indexer_rule_match_payload(
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::outcomes_reducer::reduce_indexer_rule_matches_from_outcomes;
+    use crate::types::indexer_rule_match::{ChainId, IndexerRuleMatch};
+    use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status};
+    use near_lake_framework::near_indexer_primitives::StreamerMessage;
+
+    fn read_local_file(path: &str) -> String {
+        std::fs::read_to_string(path).unwrap()
+    }
+    fn read_local_streamer_message(block_height: u64) -> StreamerMessage {
+        let path = format!("../blocks/{}.json", block_height);
+        let json = serde_json::from_str(&read_local_file(&path)).unwrap();
+        serde_json::from_value(json).unwrap()
+    }
+
+    #[tokio::test]
+    async fn match_wildcard_no_match() {
+        let wildcard_rule = IndexerRule {
+            indexer_rule_kind: IndexerRuleKind::Action,
+            matching_rule: MatchingRule::ActionAny {
+                affected_account_id: "*.nearcrow.near".to_string(),
+                status: Status::Success,
+            },
+            id: None,
+            name: None,
+        };
+
+        let streamer_message = read_local_streamer_message(93085141);
+        let result: Vec<IndexerRuleMatch> = reduce_indexer_rule_matches_from_outcomes(
+            &wildcard_rule,
+            &streamer_message,
+            ChainId::Testnet,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn match_wildcard_contract_subaccount_name() {
+        let wildcard_rule = IndexerRule {
+            indexer_rule_kind: IndexerRuleKind::Action,
+            matching_rule: MatchingRule::ActionAny {
+                affected_account_id: "*.nearcrowd.near".to_string(),
+                status: Status::Success,
+            },
+            id: None,
+            name: None,
+        };
+
+        let streamer_message = read_local_streamer_message(93085141);
+        let result: Vec<IndexerRuleMatch> = reduce_indexer_rule_matches_from_outcomes(
+            &wildcard_rule,
+            &streamer_message,
+            ChainId::Testnet,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.len(), 1); // There are two matches, until we add Extraction we are just matching the first one (block matching)
+    }
+
+    #[tokio::test]
+    async fn match_wildcard_mid_contract_name() {
+        let wildcard_rule = IndexerRule {
+            indexer_rule_kind: IndexerRuleKind::Action,
+            matching_rule: MatchingRule::ActionAny {
+                affected_account_id: "*crowd.near".to_string(),
+                status: Status::Success,
+            },
+            id: None,
+            name: None,
+        };
+
+        let streamer_message = read_local_streamer_message(93085141);
+        let result: Vec<IndexerRuleMatch> = reduce_indexer_rule_matches_from_outcomes(
+            &wildcard_rule,
+            &streamer_message,
+            ChainId::Testnet,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.len(), 1); // see Extraction note in previous test
+
+        let wildcard_rule = IndexerRule {
+            indexer_rule_kind: IndexerRuleKind::Action,
+            matching_rule: MatchingRule::ActionAny {
+                affected_account_id: "app.nea*owd.near".to_string(),
+                status: Status::Success,
+            },
+            id: None,
+            name: None,
+        };
+
+        let result: Vec<IndexerRuleMatch> = reduce_indexer_rule_matches_from_outcomes(
+            &wildcard_rule,
+            &streamer_message,
+            ChainId::Testnet,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.len(), 1); // see Extraction note in previous test
+    }
+
+    #[tokio::test]
+    async fn match_csv_account() {
+        let wildcard_rule = IndexerRule {
+            indexer_rule_kind: IndexerRuleKind::Action,
+            matching_rule: MatchingRule::ActionAny {
+                affected_account_id: "notintheblockaccount.near, app.nearcrowd.near".to_string(),
+                status: Status::Success,
+            },
+            id: None,
+            name: None,
+        };
+
+        let streamer_message = read_local_streamer_message(93085141);
+        let result: Vec<IndexerRuleMatch> = reduce_indexer_rule_matches_from_outcomes(
+            &wildcard_rule,
+            &streamer_message,
+            ChainId::Testnet,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.len(), 1); // There are two matches, until we add Extraction we are just matching the first one (block matching)
+    }
+
+    #[tokio::test]
+    async fn match_csv_wildcard_account() {
+        let wildcard_rule = IndexerRule {
+            indexer_rule_kind: IndexerRuleKind::Action,
+            matching_rule: MatchingRule::ActionAny {
+                affected_account_id: "notintheblockaccount.near, *.nearcrowd.near".to_string(),
+                status: Status::Success,
+            },
+            id: None,
+            name: None,
+        };
+
+        let streamer_message = read_local_streamer_message(93085141);
+        let result: Vec<IndexerRuleMatch> = reduce_indexer_rule_matches_from_outcomes(
+            &wildcard_rule,
+            &streamer_message,
+            ChainId::Testnet,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.len(), 1); // There are two matches, until we add Extraction we are just matching the first one (block matching)
+    }
+}

indexer/indexer_rules_engine/src/types/indexer_rule_match.rs

@@ -69,7 +69,6 @@ impl IndexerRuleMatch {
     Clone,
     Debug,
 )]
-// todo add block height
 pub enum IndexerRuleMatchPayload {
     Actions {
         block_hash: BlockHashString,

indexer/queryapi_coordinator/Cargo.toml

@@ -28,14 +28,17 @@ near-jsonrpc-client = "0.5.1"
 near-jsonrpc-primitives = "0.16.0"
 near-lake-framework = "0.7.1"
 mockall = "0.9.1"
+regex = "1"
 
 # opts
 base64 = "0.13.0"
 clap = { version = "3.1.6", features = ["derive", "env"] }
 dotenv = "0.15.0"
+tracing-subscriber = "0.2.4"
+unescape = "0.1.0"
+
+# aws
 aws-types = "0.53.0"
 aws-credential-types = "0.53.0"
 aws-sdk-s3 = "0.23.0"
 aws-sdk-sqs = "0.23.0"
-tracing-subscriber = "0.2.4"
-unescape = "0.1.0"