fix: Local development & add steps in README.md (#173)

README.md

@@ -5,7 +5,7 @@ With QueryApi you can
 * Specify the schema for your own custom hosted database and write to it with your indexer function;
 * Retrieve that data through a GraphQL API.
 
-# Table of Contents / Applications
+## 🧩 Components
 1. [QueryApi Coordinator](./indexer)
 An Indexer that tracks changes to the QueryApi registry contract. It triggers the execution of those IndexerFunctions
 when they match new blocks by placing messages on an SQS queue. Spawns historical processing threads when needed.
@@ -21,3 +21,55 @@ indexer_rules_engine, storage.
    Stores IndexerFunctions, their schemas and execution parameters like start block height.
 6. [Lake Block server](./block-server)
    Serves blocks from the S3 lake for in browser testing of IndexerFunctions.
+
+## 🚀 Getting Started
+
+The majority of the QueryApi components can be set up locally using Docker. For this purpose, a [Docker Compose file](./docker-compose.yml) has been provided. However, the local system still relies on the NEAR Mainnet, rather than running on a localnet.
+
+### Requirements
+- [Docker](https://docs.docker.com/engine/install/)
+- [Docker Compose](https://docs.docker.com/compose/install/)
+- [Hasura CLI](https://hasura.io/docs/latest/hasura-cli/install-hasura-cli/)
+- AWS Access Keys
+
+### AWS Credentials Setup
+QueryApi requires AWS credentials to stream blocks from [NEAR Lake](https://github.com/near/near-lake-indexer). Credentials are exposed via the following environment variables, which can be found in the Docker Compose file:
+
+Runner:
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+
+Coordinator:
+- `LAKE_AWS_ACCESS_KEY`
+- `LAKE_AWS_SECRET_ACCESS_KEY`
+- `QUEUE_AWS_ACCESS_KEY`
+- `QUEUE_AWS_SECRET_ACCESS_KEY`
+
+These should be populated with your credentials. In most cases, the same key pair can be used for all 3 sets of credentials. Just ensure the keys have permissions to access S3 for handling [Requestor Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html) in Near Lake. 
+
+### Hasura Configuration
+Hasura contains shared tables for e.g. logging and setting arbitrary state. These tables must be configured prior to running the entire QueryApi application. Configuration is stored in the `hasura/` directory and deployed through the Hasura CLI.
+
+To configure Hasura, first start it with:
+```sh
+docker compose up hasura-graphql --detach
+```
+
+And apply the configuration with:
+```sh
+cd ./hasura && hasura deploy
+```
+
+### Running QueryApi
+With everything configured correctly, we can now start all components of QueryApi with:
+```sh
+docker compose up
+```
+
+### Local Configuration
+- Coordinator watches the dev registry contract by default (`dev-queryapi.dataplatform.near`). To use a different contract, you can update the `REGISTRY_CONTRACT_ID` environment variable.
+- Coodinator will log SQS messages rather than sending them. To use an actual Queue, you can update the `QUEUE_URL` and `START_FROM_BLOCK_QUEUE_URL` environment variables.
+
+### Known Issues
+
+It is expected to see some provisioning errors from `Runner` when starting QueryAPI for the first time. These occur when multiple indexers under the same account attempt to provision the same shared infrastructure. These should self resolve after a few seconds.

docker-compose.yml

@@ -1,21 +1,21 @@
 version: "3.9"  # optional since v1.27.0
 services:
 
-  actions_queryapi_coordinator:
+  coordinator:
     build:
       context: ./indexer
       args:
-        - chain_id=mainnet
-    links:
+        - CARGO_BUILD_MODE=debug
+    depends_on:
       - redis
     environment:
       REDIS_CONNECTION_STRING: redis://redis
-      LAKE_AWS_SECRET_ACCESS_KEY: 
-      LAKE_AWS_ACCESS_KEY: 
-      QUEUE_AWS_ACCESS_KEY: 
-      QUEUE_AWS_SECRET_ACCESS_KEY: 
-      QUEUE_URL:
-      START_FROM_BLOCK_QUEUE_URL:
+      LAKE_AWS_ACCESS_KEY:
+      LAKE_AWS_SECRET_ACCESS_KEY:
+      QUEUE_AWS_ACCESS_KEY:
+      QUEUE_AWS_SECRET_ACCESS_KEY:
+      QUEUE_URL: MOCK
+      START_FROM_BLOCK_QUEUE_URL: MOCK
       PORT: 9180
       REGISTRY_CONTRACT_ID: dev-queryapi.dataplatform.near
       AWS_QUEUE_REGION: eu-central-1
@@ -27,7 +27,7 @@ services:
     build:
       context: ./runner
     depends_on:
-    - "graphql-engine"
+    - "hasura-graphql"
     - "redis"
     environment:
       REGION: eu-central-1
@@ -39,8 +39,9 @@ services:
       PGUSER: postgres
       PGPASSWORD: postgrespassword
       PGDATABASE: postgres
-      AWS_ACCESS_KEY_ID: 
-      AWS_SECRET_ACCESS_KEY: 
+      PORT: 9180
+      AWS_ACCESS_KEY_ID:
+      AWS_SECRET_ACCESS_KEY:
 
   redis:
     image: redis
@@ -49,15 +50,15 @@ services:
       - "--save 60 1"
       - "--loglevel warning"
     volumes:
-      - ./redis/data:/data
+      - redis:/data
     ports:
       - "6379:6379"
 
   postgres:
     image: postgres:12
     restart: always
     volumes:
-    - db_data:/var/lib/postgresql/data
+    - postgres:/var/lib/postgresql/data
     environment:
       POSTGRES_PASSWORD: postgrespassword
     ports:
@@ -68,8 +69,6 @@ services:
       context: ./hasura-authentication-service
     ports:
     - "4000:4000"
-    depends_on:
-    - "hasura-graphql"
     environment:
       PORT: 4000
       DEFAULT_HASURA_ROLE: append
@@ -80,6 +79,7 @@ services:
     - "8080:8080"
     depends_on:
     - "postgres"
+    - "hasura-auth"
     restart: always
     environment:
       HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
@@ -90,4 +90,5 @@ services:
       HASURA_GRAPHQL_AUTH_HOOK: http://hasura-auth:4000/auth
 
 volumes:
-  db_data:
+  postgres:
+  redis:

indexer/Dockerfile

@@ -1,30 +1,16 @@
 FROM rust:1.68 AS build
-
+ARG CARGO_BUILD_MODE=release
 WORKDIR /tmp/
-COPY Cargo.toml Cargo.lock ./
-COPY storage/Cargo.toml ./storage/
-COPY indexer_rule_type/Cargo.toml ./indexer_rule_type/
-COPY indexer_rules_engine/Cargo.toml ./indexer_rules_engine/
-COPY queryapi_coordinator/Cargo.toml ./queryapi_coordinator/
-
-# We have to use sparse-registry cargo feature to avoid running out of RAM (version 1.68+)
-ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
-RUN /bin/bash -c "mkdir -p {queryapi_coordinator,indexer_rule_type,indexer_rules_engine,storage}/src" && \
-    echo 'fn main() {}' > queryapi_coordinator/src/main.rs && \
-    touch indexer_rule_type/src/lib.rs && \
-    touch indexer_rules_engine/src/lib.rs && \
-    touch storage/src/lib.rs && \
-    cargo build
-
 COPY ./ ./
-
-RUN cargo build --release --package queryapi_coordinator --offline
-
+RUN if [ "$CARGO_BUILD_MODE" = "debug" ]; then \
+        cargo build --package queryapi_coordinator --offline; \
+    else \
+        cargo build --release --package queryapi_coordinator --offline; \
+    fi
 
 FROM ubuntu:20.04
-
+ARG CARGO_BUILD_MODE=release
 RUN apt update && apt install -yy openssl ca-certificates
-
 USER nobody
-COPY --from=build /tmp/target/release/queryapi_coordinator /queryapi_coordinator
+COPY --from=build /tmp/target/$CARGO_BUILD_MODE/queryapi_coordinator /queryapi_coordinator
 ENTRYPOINT ["/queryapi_coordinator"]

runner/src/index.ts

@@ -11,7 +11,7 @@ metrics.startServer().catch((err) => {
 });
 
 // const BATCH_SIZE = 1;
-const STREAM_START_ID = '0';
+const STREAM_SMALLEST_ID = '0';
 // const STREAM_THROTTLE_MS = 250;
 const STREAM_HANDLER_THROTTLE_MS = 500;
 
@@ -63,7 +63,7 @@ const getMessagesFromStream = async <Message extends Record<string, string>>(
   lastId: string | null,
   count: number,
 ): Promise<StreamMessages<Message> | null> => {
-  const id = lastId ?? STREAM_START_ID;
+  const id = lastId ?? STREAM_SMALLEST_ID;
 
   const results = await client.xRead(
     { key: generateStreamKey(indexerName), id },
@@ -74,13 +74,17 @@ const getMessagesFromStream = async <Message extends Record<string, string>>(
   return results?.[0].messages as StreamMessages<Message>;
 };
 
+const incrementStreamId = (id: string): string => {
+  const [timestamp, sequenceNumber] = id.split('-');
+  const nextSequenceNumber = Number(sequenceNumber) + 1;
+  return `${timestamp}-${nextSequenceNumber}`;
+};
+
 const getUnprocessedMessages = async <Message extends Record<string, string>>(
   indexerName: string,
-  startId: string
+  startId: string | null
 ): Promise<Array<StreamMessage<Message>>> => {
-  const [timestamp, sequenceNumber] = startId.split('-');
-  const nextSequenceNumber = Number(sequenceNumber) + 1;
-  const nextId = `${timestamp}-${nextSequenceNumber}`;
+  const nextId = startId ? incrementStreamId(startId) : STREAM_SMALLEST_ID;
 
   const results = await client.xRange(generateStreamKey(indexerName), nextId, '+');
 
@@ -147,7 +151,7 @@ const processStream = async (indexerName: string): Promise<void> => {
 
       metrics.EXECUTION_DURATION.labels({ indexer: indexerName }).set(endTime - startTime);
 
-      const unprocessedMessages = await getUnprocessedMessages<IndexerStreamMessage>(indexerName, lastProcessedId ?? '-');
+      const unprocessedMessages = await getUnprocessedMessages<IndexerStreamMessage>(indexerName, lastProcessedId);
       metrics.UNPROCESSED_STREAM_MESSAGES.labels({ indexer: indexerName }).set(unprocessedMessages?.length ?? 0);
 
       console.log(`Success: ${indexerName}`);