feat: Query Hasura Endpoint through Block Streamer

block-streamer/Cargo.toml

@@ -13,11 +13,15 @@ borsh = "0.10.2"
 cached = "0.49.3"
 chrono = "0.4.25"
 futures = "0.3.5"
+graphql_client = "0.14.0"
 lazy_static = "1.4.0"
 mockall = "0.11.4"
+near-lake-framework = "0.7.8"
 prometheus = "0.13.3"
 prost = "0.12.3"
 redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] }
+registry-types = { path = "../registry/types" }
+reqwest = { version = "^0.12.4", features = ["json", "blocking"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1.0.55"
 tracing = "0.1.40"
@@ -29,10 +33,6 @@ tokio-stream = "0.1.14"
 tonic = "0.10.2"
 wildmatch = "2.1.1"
 
-registry-types = { path = "../registry/types" }
-
-near-lake-framework = "0.7.8"
-
 [build-dependencies]
 tonic-build = "0.10"
 

block-streamer/README.md

@@ -0,0 +1,14 @@
+// TODO: Improve README further
+
+## GraphQL Code Generation
+Querying a GraphQL requires informing Rust of the correct types to deserialize the response data into. In order to do this, the schema of the GraphQL data needs to be introspected. Following that, the query intended to be called needs to be fully defined. With this information, code can be generated using the graphql-client API. Below are the instructions on how to do so. 
+
+### Generating schema.json
+Run the following command with the relevant sections replaced. It will create a JSON containing schemas for ALL tables under some Hasura Role.
+
+`graphql-client introspect-schema --output PATH_TO_SOMEWHERE HASURA_ENDPOINT/v1/graphql --header 'x-hasura-role: SOME_HASURA_ROLE'`
+
+### Generating Rust types file for query
+Run the following command with the correct arguments to generate a Rust file containing Structs and Modules to deserialize GraphQL responses for that particular query. After the codegen completes, you may need to manually modify the file further to resolve type issues. For example, replacing `super::date` with `String`.
+
+`graphql-client generate --schema-path PATH_TO_SCHEMA_JSON --response-derives 'Debug' --output-directory PATH_TO_GRAPHQL_QUERIES_FOLDER PATH_TO_QUERY_GRAPHQL_FILE`

block-streamer/src/graphql/client.rs

@@ -0,0 +1,126 @@
+use crate::graphql::queries::get_bitmaps_exact::{get_bitmaps_exact, GetBitmapsExact};
+use crate::graphql::queries::get_bitmaps_wildcard::{get_bitmaps_wildcard, GetBitmapsWildcard};
+use ::reqwest;
+use graphql_client::{GraphQLQuery, Response};
+use std::error::Error;
+
+// TODO: Use Dataplatform account
+const HASURA_ACCOUNT: &str = "darunrs_near";
+
+pub struct GraphqlClient {
+    client: reqwest::Client,
+    graphql_endpoint: String,
+}
+
+#[cfg_attr(test, mockall::automock)]
+impl GraphqlClient {
+    pub fn new(graphql_endpoint: String) -> Self {
+        Self {
+            client: reqwest::Client::new(),
+            graphql_endpoint,
+        }
+    }
+
+    pub async fn get_bitmaps_exact(
+        &self,
+        receiver_ids: Vec<String>,
+        block_date: String,
+        limit: i64,
+        offset: i64,
+    ) -> Result<
+        Vec<get_bitmaps_exact::GetBitmapsExactDarunrsNearBitmapV5ActionsIndex>,
+        Box<dyn Error>,
+    > {
+        let variables = get_bitmaps_exact::Variables {
+            receiver_ids: Some(receiver_ids),
+            block_date: Some(block_date),
+            limit: Some(limit),
+            offset: Some(offset),
+        };
+        let request_body = GetBitmapsExact::build_query(variables);
+        let res = self
+            .client
+            .post(&self.graphql_endpoint)
+            .header("x-hasura-role", HASURA_ACCOUNT)
+            .json(&request_body)
+            .send()
+            .await
+            .expect("Failed to query bitmaps for list of exact receivers");
+        let response_body: Response<get_bitmaps_exact::ResponseData> = res.json().await?;
+        match response_body.data {
+            Some(data) => Ok(data.darunrs_near_bitmap_v5_actions_index),
+            None => Ok([].into()),
+        }
+    }
+
+    pub async fn get_bitmaps_wildcard(
+        &self,
+        receiver_ids: String,
+        block_date: String,
+        limit: i64,
+        offset: i64,
+    ) -> Result<
+        Vec<get_bitmaps_wildcard::GetBitmapsWildcardDarunrsNearBitmapV5ActionsIndex>,
+        Box<dyn Error>,
+    > {
+        let variables = get_bitmaps_wildcard::Variables {
+            receiver_ids: Some(receiver_ids),
+            block_date: Some(block_date),
+            limit: Some(limit),
+            offset: Some(offset),
+        };
+        let request_body = GetBitmapsWildcard::build_query(variables);
+        let res = self
+            .client
+            .post(&self.graphql_endpoint)
+            .header("x-hasura-role", HASURA_ACCOUNT)
+            .json(&request_body)
+            .send()
+            .await
+            .expect("Failed to query bitmaps for wildcard receivers");
+        let response_body: Response<get_bitmaps_wildcard::ResponseData> = res.json().await?;
+        match response_body.data {
+            Some(data) => Ok(data.darunrs_near_bitmap_v5_actions_index),
+            None => Ok([].into()),
+        }
+    }
+}
+
+// TODO: Remove Unit tests after bitmap query is integrated into the main application
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const HASURA_ENDPOINT: &str =
+        "https://queryapi-hasura-graphql-mainnet-vcqilefdcq-ew.a.run.app/v1/graphql";
+
+    #[tokio::test]
+    async fn test_get_bitmaps_exact() {
+        let client = GraphqlClient::new(HASURA_ENDPOINT.to_string());
+        let receiver_ids = vec!["app.nearcrowd.near".to_string()];
+        let block_date = "2024-03-21".to_string();
+        let limit = 10;
+        let offset = 0;
+        let response = client
+            .get_bitmaps_exact(receiver_ids, block_date, limit, offset)
+            .await
+            .unwrap();
+        assert_eq!(response[0].first_block_height, 115130287);
+    }
+
+    // This query takes several seconds
+    #[ignore]
+    #[tokio::test]
+    async fn test_get_bitmaps_wildcard() {
+        let client = GraphqlClient::new(HASURA_ENDPOINT.to_string());
+        let receiver_ids = "app.nearcrowd.near".to_string();
+        let block_date = "2024-03-21".to_string();
+        let limit = 10;
+        let offset = 0;
+        let response = client
+            .get_bitmaps_wildcard(receiver_ids, block_date, limit, offset)
+            .await
+            .unwrap();
+        assert_eq!(response[0].first_block_height, 115130287);
+    }
+}

block-streamer/src/graphql/mod.rs

@@ -0,0 +1,2 @@
+pub mod client;
+pub mod queries;

block-streamer/src/graphql/queries/get_bitmaps_exact.rs

@@ -0,0 +1,49 @@
+// TODO: Replace this file with query agaisnt production Indexer
+#![allow(clippy::all, warnings)]
+pub struct GetBitmapsExact;
+pub mod get_bitmaps_exact {
+    #![allow(dead_code)]
+    use std::result::Result;
+    pub const OPERATION_NAME: &str = "GetBitmapsExact";
+    pub const QUERY : & str = "query GetBitmapsExact($block_date: date, $receiver_ids: [String!], $limit: Int, $offset: Int) {\n  darunrs_near_bitmap_v5_actions_index(limit: $limit, offset: $offset, where: {block_date: {_eq: $block_date}, receiver: {receiver: {_in: $receiver_ids}}}) {\n    bitmap\n    first_block_height\n  }\n}" ;
+    use super::*;
+    use serde::{Deserialize, Serialize};
+    #[allow(dead_code)]
+    type Boolean = bool;
+    #[allow(dead_code)]
+    type Float = f64;
+    #[allow(dead_code)]
+    type Int = i64;
+    #[allow(dead_code)]
+    type ID = String;
+    type date = String;
+    #[derive(Serialize)]
+    pub struct Variables {
+        pub block_date: Option<date>,
+        pub receiver_ids: Option<Vec<String>>,
+        pub limit: Option<Int>,
+        pub offset: Option<Int>,
+    }
+    impl Variables {}
+    #[derive(Deserialize, Debug)]
+    pub struct ResponseData {
+        pub darunrs_near_bitmap_v5_actions_index:
+            Vec<GetBitmapsExactDarunrsNearBitmapV5ActionsIndex>,
+    }
+    #[derive(Deserialize, Debug)]
+    pub struct GetBitmapsExactDarunrsNearBitmapV5ActionsIndex {
+        pub bitmap: String,
+        pub first_block_height: Int,
+    }
+}
+impl graphql_client::GraphQLQuery for GetBitmapsExact {
+    type Variables = get_bitmaps_exact::Variables;
+    type ResponseData = get_bitmaps_exact::ResponseData;
+    fn build_query(variables: Self::Variables) -> ::graphql_client::QueryBody<Self::Variables> {
+        graphql_client::QueryBody {
+            variables,
+            query: get_bitmaps_exact::QUERY,
+            operation_name: get_bitmaps_exact::OPERATION_NAME,
+        }
+    }
+}

block-streamer/src/graphql/queries/get_bitmaps_wildcard.rs

@@ -0,0 +1,49 @@
+// TODO: Replace this file with query agaisnt production Indexer
+#![allow(clippy::all, warnings)]
+pub struct GetBitmapsWildcard;
+pub mod get_bitmaps_wildcard {
+    #![allow(dead_code)]
+    use std::result::Result;
+    pub const OPERATION_NAME: &str = "GetBitmapsWildcard";
+    pub const QUERY : & str = "query GetBitmapsWildcard($block_date: date, $receiver_ids: String, $limit: Int, $offset: Int) {\n  darunrs_near_bitmap_v5_actions_index(limit: $limit, offset: $offset, where: {block_date: {_eq: $block_date}, receiver: {receiver: {_regex: $receiver_ids}}}) {\n    bitmap\n    first_block_height\n  }\n}" ;
+    use super::*;
+    use serde::{Deserialize, Serialize};
+    #[allow(dead_code)]
+    type Boolean = bool;
+    #[allow(dead_code)]
+    type Float = f64;
+    #[allow(dead_code)]
+    type Int = i64;
+    #[allow(dead_code)]
+    type ID = String;
+    type date = String;
+    #[derive(Serialize)]
+    pub struct Variables {
+        pub block_date: Option<date>,
+        pub receiver_ids: Option<String>,
+        pub limit: Option<Int>,
+        pub offset: Option<Int>,
+    }
+    impl Variables {}
+    #[derive(Deserialize, Debug)]
+    pub struct ResponseData {
+        pub darunrs_near_bitmap_v5_actions_index:
+            Vec<GetBitmapsWildcardDarunrsNearBitmapV5ActionsIndex>,
+    }
+    #[derive(Deserialize, Debug)]
+    pub struct GetBitmapsWildcardDarunrsNearBitmapV5ActionsIndex {
+        pub bitmap: String,
+        pub first_block_height: Int,
+    }
+}
+impl graphql_client::GraphQLQuery for GetBitmapsWildcard {
+    type Variables = get_bitmaps_wildcard::Variables;
+    type ResponseData = get_bitmaps_wildcard::ResponseData;
+    fn build_query(variables: Self::Variables) -> ::graphql_client::QueryBody<Self::Variables> {
+        graphql_client::QueryBody {
+            variables,
+            query: get_bitmaps_wildcard::QUERY,
+            operation_name: get_bitmaps_wildcard::OPERATION_NAME,
+        }
+    }
+}

block-streamer/src/graphql/queries/mod.rs

@@ -0,0 +1,2 @@
+pub mod get_bitmaps_exact;
+pub mod get_bitmaps_wildcard;

block-streamer/src/main.rs

@@ -2,6 +2,7 @@ use tracing_subscriber::prelude::*;
 
 mod block_stream;
 mod delta_lake_client;
+mod graphql;
 mod indexer_config;
 mod lake_s3_client;
 mod metrics;