Prod deploy 06/10/23

block-server/handler.js

@@ -1,6 +1,6 @@
 'use strict';
-const AWS = require('aws-sdk');
-const S3= new AWS.S3();
+import { S3 } from '@aws-sdk/client-s3';
+const S3 = new S3();
 
 const NETWORK = process.env.NETWORK || 'mainnet';
 

frontend/replacement.dev.json

@@ -1,6 +1,7 @@
 {
     "REPL_ACCOUNT_ID": "dev-queryapi.dataplatform.near",
     "REPL_GRAPHQL_ENDPOINT": "https://near-queryapi.dev.api.pagoda.co",
+    "REPL_GRAPHQL_ENDPOINT_V2": "https://queryapi-hasura-graphql-mainnet-vcqilefdcq-ew.a.run.app",
     "REPL_EXTERNAL_APP_URL": "https://queryapi-frontend-vcqilefdcq-ew.a.run.app",
     "REPL_REGISTRY_CONTRACT_ID": "dev-queryapi.dataplatform.near"
 }

frontend/replacement.local.json

@@ -1,6 +1,7 @@
 {
     "REPL_ACCOUNT_ID": "dataplatform.near",
     "REPL_GRAPHQL_ENDPOINT": "https://near-queryapi.api.pagoda.co",
+    "REPL_GRAPHQL_ENDPOINT_V2": "https://queryapi-hasura-graphql-mainnet-vcqilefdcq-ew.a.run.app",
     "REPL_EXTERNAL_APP_URL": "http://localhost:3000",
     "REPL_REGISTRY_CONTRACT_ID": "queryapi.dataplatform.near"
 }

frontend/replacement.mainnet.json

@@ -1,6 +1,7 @@
 {
     "REPL_ACCOUNT_ID": "dataplatform.near",
     "REPL_GRAPHQL_ENDPOINT": "https://near-queryapi.api.pagoda.co",
+    "REPL_GRAPHQL_ENDPOINT_V2": "https://queryapi-hasura-graphql-mainnet-24ktefolwq-ew.a.run.app",
     "REPL_EXTERNAL_APP_URL": "https://queryapi-frontend-24ktefolwq-ew.a.run.app",
     "REPL_REGISTRY_CONTRACT_ID": "queryapi.dataplatform.near"
 }

frontend/widgets/src/QueryApi.IndexerStatus.jsx

@@ -95,6 +95,9 @@ const TextLink = styled.a`
 
 if (!indexer_name) return "missing indexer_name";
 
+let v1_endpoint = `${REPL_GRAPHQL_ENDPOINT}`;
+let v2_endpoint = `${REPL_GRAPHQL_ENDPOINT_V2}`;
+
 State.init({
   logs: [],
   state: [],
@@ -105,11 +108,17 @@ State.init({
   indexer_resPage: 0,
   logsPage: 0,
   statePage: 0,
+  v2Toggle: Storage.privateGet("QueryApiV2Toggle") || false,
 });
 
+let graphQLEndpoint = state.v2Toggle ? v2_endpoint : v1_endpoint;
+
 function fetchGraphQL(operationsDoc, operationName, variables) {
-  return asyncFetch(`${REPL_GRAPHQL_ENDPOINT}/v1/graphql`, {
+  return asyncFetch(`${graphQLEndpoint}/v1/graphql`, {
     method: "POST",
+    headers: {
+      "x-hasura-role": "append"
+    },
     body: JSON.stringify({
       query: operationsDoc,
       variables: variables,
@@ -119,7 +128,7 @@ function fetchGraphQL(operationsDoc, operationName, variables) {
 }
 
 const createGraphQLLink = () => {
-  const queryLink = `https://cloud.hasura.io/public/graphiql?endpoint=${REPL_GRAPHQL_ENDPOINT}/v1/graphql&query=query+IndexerQuery+%7B%0A++indexer_state%28where%3A+%7Bfunction_name%3A+%7B_eq%3A+%22function_placeholder%22%7D%7D%29+%7B%0A++++function_name%0A++++current_block_height%0A++%7D%0A++indexer_log_entries%28%0A++++where%3A+%7Bfunction_name%3A+%7B_eq%3A+%22function_placeholder%22%7D%7D%0A++++order_by%3A+%7B+timestamp%3A+desc%7D%0A++%29+%7B%0A++++function_name%0A++++id%0A++++message%0A++++timestamp%0A++%7D%0A%7D%0A`;
+  const queryLink = `https://cloud.hasura.io/public/graphiql?endpoint=${graphQLEndpoint}/v1/graphql&query=query+IndexerQuery+%7B%0A++indexer_state%28where%3A+%7Bfunction_name%3A+%7B_eq%3A+%22function_placeholder%22%7D%7D%29+%7B%0A++++function_name%0A++++current_block_height%0A++%7D%0A++indexer_log_entries%28%0A++++where%3A+%7Bfunction_name%3A+%7B_eq%3A+%22function_placeholder%22%7D%7D%0A++++order_by%3A+%7B+timestamp%3A+desc%7D%0A++%29+%7B%0A++++function_name%0A++++id%0A++++message%0A++++timestamp%0A++%7D%0A%7D%0A`;
   return queryLink.replaceAll(
     "function_placeholder",
     `${accountId}/${indexer_name}`
@@ -152,14 +161,15 @@ const indexerStateDoc = `
       current_block_height
       current_historical_block_height
     }
-    indexer_state_aggregate(where: {function_name: {_eq: "${accountId}/${indexer_name}"}}) {
-    aggregate {
-      count
-    }
-    }
   }
 `;
-if (!state.initialFetch) {
+
+const prevV2ToggleSelected = Storage.privateGet("QueryApiV2Toggle");
+if (
+  !state.initialFetch ||
+  (prevV2ToggleSelected !== state.v2Toggle)
+) {
+  Storage.privateSet("QueryApiV2Toggle", state.v2Toggle);
   fetchGraphQL(logsDoc, "QueryLogs", {
     offset: state.logsPage * LIMIT,
   }).then((result) => {
@@ -235,6 +245,44 @@ const onIndexerResPageChange = (page) => {
   State.update({ indexer_resPage: page, currentPage: page });
 };
 
+const DisclaimerContainer = styled.div`
+  padding: 10px;
+  margin: 0.5px;
+  text-color: black;
+  display: flex;
+  width: 50;
+  border: 2px solid rgb(240, 240, 240);
+  border-radius: 8px;
+  align-items: "center";
+  margin-bottom: 5px;
+`;
+
+const Notice = styled.div`
+  font-weight: 900;
+  font-size: 22px;
+  align-self: flex-start;
+  margin: 10px 0px 30px;
+  text-align: center;
+  padding-bottom: 5px;
+  border-bottom: 1px solid rgb(240, 240, 241);
+  color: rgb(36, 39, 42);
+`;
+
+const DisclaimerText = styled.p`
+  font-size: 14px;
+  line-height: 20px;
+  font-weight: 400;
+  color: rgb(17, 24, 28);
+  word-break: break-word;
+  width: 700px;
+  text-align: start;
+  padding-left: 10px;
+
+  @media (max-width: 1024px) {
+    width: 80%;
+  }
+`;
+
 return (
   <>
     <Card>
@@ -244,6 +292,39 @@ return (
           GraphQL Playground
           <i className="bi bi-box-arrow-up-right"></i>
         </TextLink>
+        <div
+          style={{
+            marginTop: "5px",
+            display: "flex",
+            width: "100%",
+            justifyContent: "center",
+          }}
+        >
+          <DisclaimerContainer>
+            <div className="flex">
+              <Notice>V2 Testing Notice</Notice>
+              <div style={{ display: "flex" }}>
+                <DisclaimerText>
+                  QueryAPI is still in beta. We are working on a OueryAPI V2
+                  with faster historical processing, easier access to DB and
+                  more control over your indexer. V2 is running in parallel and
+                  you can see the logs from this new version by toggling this
+                  button.
+                </DisclaimerText>
+                <Widget
+                  src={`${REPL_ACCOUNT_ID}/widget/components.toggle`}
+                  props={{
+                    active: state.v2Toggle,
+                    label: "",
+                    onSwitch: () => {
+                      State.update({ v2Toggle: !state.v2Toggle });
+                    },
+                  }}
+                />
+              </div>
+            </div>
+          </DisclaimerContainer>
+        </div>
       </Title>
 
       <CardBody>
@@ -270,7 +351,9 @@ return (
                   <tr>
                     <TableElement>{x.function_name}</TableElement>
                     <TableElement>{x.current_block_height}</TableElement>
-                    <TableElement>{x.current_historical_block_height}</TableElement>
+                    <TableElement>
+                      {x.current_historical_block_height}
+                    </TableElement>
                     <TableElement>{x.status}</TableElement>
                   </tr>
                 ))}

frontend/widgets/src/components/toggle.jsx

@@ -0,0 +1,78 @@
+const ToggleRoot = styled.div`
+  justify-content: space-between;
+  width: fit-content;
+  max-width: 100%;
+`;
+
+const ToggleSwitchRoot = styled("Switch.Root")`
+  all: unset;
+  display: block;
+  width: 42px;
+  height: 25px;
+  background-color: #d1d1d1;
+  border-radius: 9999px;
+  position: relative;
+  box-shadow: 0 2px 10px var(--blackA7);
+
+  &[data-state="checked"] {
+    background-color: #00d084;
+  }
+
+  &[data-disabled=""] {
+    opacity: 0.7;
+  }
+`;
+
+const ToggleSwitchThumb = styled("Switch.Thumb")`
+  all: unset;
+  display: block;
+  width: 21px;
+  height: 21px;
+  border-radius: 9999px;
+  transition: transform 100ms;
+  transform: translateX(2px);
+  will-change: transform;
+
+  &[data-state="checked"] {
+    transform: translateX(19px);
+  }
+`;
+
+const ToggleLabel = styled.label`
+  white-space: nowrap;
+`;
+
+const Toggle = ({
+  active,
+  className,
+  direction,
+  disabled,
+  key,
+  label,
+  onSwitch,
+  ...rest
+}) => (
+  <ToggleRoot
+    className={[
+      "d-flex justify-content-between, align-items-center gap-3",
+      direction === "rtl" ? "flex-row-reverse" : "",
+      className,
+    ].join(" ")}
+    {...rest}
+  >
+    <ToggleLabel htmlFor={`toggle-${key}`}>{label}</ToggleLabel>
+
+    <ToggleSwitchRoot
+      checked={active}
+      className="shadow-none"
+      id={`toggle-${key}`}
+      onCheckedChange={disabled ? null : onSwitch}
+      title={disabled ? `Permanently ${active ? "enabled" : "disabled"}` : null}
+      {...{ disabled }}
+    >
+      {!disabled && <ToggleSwitchThumb className="bg-light shadow" />}
+    </ToggleSwitchRoot>
+  </ToggleRoot>
+);
+
+return Toggle(props);

hasura/metadata/databases/default/tables/public_indexer_log_entries.yaml

@@ -19,5 +19,6 @@ select_permissions:
     - message
     - timestamp
     - id
+    allow_aggregations: true
     filter: {}
   role: append

hasura/metadata/databases/default/tables/public_indexer_state.yaml

@@ -17,6 +17,7 @@ select_permissions:
     - current_block_height
     - current_historical_block_height
     - status
+    allow_aggregations: true
     filter: {}
   role: append
 update_permissions:

indexer/queryapi_coordinator/src/historical_block_processing.rs

@@ -151,6 +151,7 @@ pub(crate) async fn process_historical_messages(
                     redis_connection_manager,
                     storage::generate_historical_storage_key(&indexer_function.get_full_name()),
                     serde_json::to_string(&indexer_function)?,
+                    None,
                 )
                 .await?;
             }

indexer/queryapi_coordinator/src/main.rs

@@ -6,11 +6,12 @@ use tokio::sync::{Mutex, MutexGuard};
 use indexer_rules_engine::types::indexer_rule_match::{ChainId, IndexerRuleMatch};
 use near_lake_framework::near_indexer_primitives::types::{AccountId, BlockHeight};
 use near_lake_framework::near_indexer_primitives::{types, StreamerMessage};
+use utils::serialize_to_camel_case_json_string;
 
 use crate::indexer_types::IndexerFunction;
 use indexer_types::{IndexerQueueMessage, IndexerRegistry};
 use opts::{Opts, Parser};
-use storage::{self, ConnectionManager};
+use storage::{self, generate_real_time_streamer_message_key, ConnectionManager};
 
 mod historical_block_processing;
 mod indexer_reducer;
@@ -146,6 +147,15 @@ async fn handle_streamer_message(
 
     let block_height: BlockHeight = context.streamer_message.block.header.height;
 
+    // Cache streamer message block and shards for use in real time processing
+    storage::set(
+        context.redis_connection_manager,
+        generate_real_time_streamer_message_key(block_height),
+        &serialize_to_camel_case_json_string(&context.streamer_message)?,
+        Some(60),
+    )
+    .await?;
+
     let spawned_indexers = indexer_registry::index_registry_changes(
         block_height,
         &mut indexer_registry_locked,
@@ -206,6 +216,7 @@ async fn handle_streamer_message(
                     context.redis_connection_manager,
                     storage::generate_real_time_storage_key(&indexer_function.get_full_name()),
                     serde_json::to_string(indexer_function)?,
+                    None,
                 )
                 .await?;
                 storage::xadd(

indexer/queryapi_coordinator/src/utils.rs

@@ -1,3 +1,5 @@
+use serde_json::Value;
+
 pub(crate) async fn stats(redis_connection_manager: storage::ConnectionManager) {
     let interval_secs = 10;
     let mut previous_processed_blocks: u64 =
@@ -49,3 +51,52 @@ pub(crate) async fn stats(redis_connection_manager: storage::ConnectionManager)
         tokio::time::sleep(std::time::Duration::from_secs(interval_secs)).await;
     }
 }
+
+pub(crate) fn serialize_to_camel_case_json_string(
+    streamer_message: &near_lake_framework::near_indexer_primitives::StreamerMessage,
+) -> anyhow::Result<String, serde_json::Error> {
+    // Serialize the Message object to a JSON string
+    let json_str = serde_json::to_string(&streamer_message)?;
+
+    // Deserialize the JSON string to a Value Object
+    let mut message_value: Value = serde_json::from_str(&json_str)?;
+
+    // Convert keys to Camel Case
+    to_camel_case_keys(&mut message_value);
+
+    return serde_json::to_string(&message_value);
+}
+
+fn to_camel_case_keys(message_value: &mut Value) {
+    // Only process if subfield contains objects
+    match message_value {
+        Value::Object(map) => {
+            for key in map.keys().cloned().collect::<Vec<String>>() {
+                // Generate Camel Case Key
+                let new_key = key
+                    .split("_")
+                    .enumerate()
+                    .map(|(i, str)| {
+                        if i > 0 {
+                            return str[..1].to_uppercase() + &str[1..];
+                        }
+                        return str.to_owned();
+                    })
+                    .collect::<Vec<String>>()
+                    .join("");
+
+                // Recursively process inner fields and update map with new key
+                if let Some(mut val) = map.remove(&key) {
+                    to_camel_case_keys(&mut val);
+                    map.insert(new_key, val);
+                }
+            }
+        }
+        Value::Array(vec) => {
+            for val in vec {
+                to_camel_case_keys(val);
+            }
+        }
+        _ => {}
+    }
+}