From 15a54a339d0a22c917e7d8882a520124f8ea622b Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 16:51:04 +1200 Subject: [PATCH 1/8] fix: Ensure initial id in `getUnprocessedMessages` is valid --- runner/src/index.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/runner/src/index.ts b/runner/src/index.ts index b6a405ac4..66bbc10cb 100644 --- a/runner/src/index.ts +++ b/runner/src/index.ts @@ -11,7 +11,7 @@ metrics.startServer().catch((err) => { }); // const BATCH_SIZE = 1; -const STREAM_START_ID = '0'; +const STREAM_SMALLEST_ID = '0'; // const STREAM_THROTTLE_MS = 250; const STREAM_HANDLER_THROTTLE_MS = 500; @@ -63,7 +63,7 @@ const getMessagesFromStream = async >( lastId: string | null, count: number, ): Promise | null> => { - const id = lastId ?? STREAM_START_ID; + const id = lastId ?? STREAM_SMALLEST_ID; const results = await client.xRead( { key: generateStreamKey(indexerName), id }, @@ -74,13 +74,17 @@ const getMessagesFromStream = async >( return results?.[0].messages as StreamMessages; }; +const incrementStreamId = (id: string): string => { + const [timestamp, sequenceNumber] = id.split('-'); + const nextSequenceNumber = Number(sequenceNumber) + 1; + return `${timestamp}-${nextSequenceNumber}`; +}; + const getUnprocessedMessages = async >( indexerName: string, - startId: string + startId: string | null ): Promise>> => { - const [timestamp, sequenceNumber] = startId.split('-'); - const nextSequenceNumber = Number(sequenceNumber) + 1; - const nextId = `${timestamp}-${nextSequenceNumber}`; + const nextId = startId ? incrementStreamId(startId) : STREAM_SMALLEST_ID; const results = await client.xRange(generateStreamKey(indexerName), nextId, '+'); @@ -147,7 +151,7 @@ const processStream = async (indexerName: string): Promise => { metrics.EXECUTION_DURATION.labels({ indexer: indexerName }).set(endTime - startTime); - const unprocessedMessages = await getUnprocessedMessages(indexerName, lastProcessedId ?? '-'); + const unprocessedMessages = await getUnprocessedMessages(indexerName, lastProcessedId); metrics.UNPROCESSED_STREAM_MESSAGES.labels({ indexer: indexerName }).set(unprocessedMessages?.length ?? 0); console.log(`Success: ${indexerName}`); From 38338a293c319f9631c59e94d71ac07a52123aed Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 16:52:27 +1200 Subject: [PATCH 2/8] chore: Remove cargo pre-build in docker as layers arent cached --- indexer/Dockerfile | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/indexer/Dockerfile b/indexer/Dockerfile index c8693071f..1669a505e 100644 --- a/indexer/Dockerfile +++ b/indexer/Dockerfile @@ -1,21 +1,6 @@ FROM rust:1.68 AS build WORKDIR /tmp/ -COPY Cargo.toml Cargo.lock ./ -COPY storage/Cargo.toml ./storage/ -COPY indexer_rule_type/Cargo.toml ./indexer_rule_type/ -COPY indexer_rules_engine/Cargo.toml ./indexer_rules_engine/ -COPY queryapi_coordinator/Cargo.toml ./queryapi_coordinator/ - -# We have to use sparse-registry cargo feature to avoid running out of RAM (version 1.68+) -ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse -RUN /bin/bash -c "mkdir -p {queryapi_coordinator,indexer_rule_type,indexer_rules_engine,storage}/src" && \ - echo 'fn main() {}' > queryapi_coordinator/src/main.rs && \ - touch indexer_rule_type/src/lib.rs && \ - touch indexer_rules_engine/src/lib.rs && \ - touch storage/src/lib.rs && \ - cargo build - COPY ./ ./ RUN cargo build --release --package queryapi_coordinator --offline From d3c46ab326dffbfb09c7b2db5b038f577e560051 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 16:53:06 +1200 Subject: [PATCH 3/8] feat: Use docker build arg to toggle cargo debug/release build --- indexer/Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/indexer/Dockerfile b/indexer/Dockerfile index 1669a505e..1c9ecdaef 100644 --- a/indexer/Dockerfile +++ b/indexer/Dockerfile @@ -1,15 +1,16 @@ FROM rust:1.68 AS build - +ARG CARGO_BUILD_MODE=release WORKDIR /tmp/ COPY ./ ./ - -RUN cargo build --release --package queryapi_coordinator --offline - +RUN if [ "$CARGO_BUILD_MODE" = "debug" ]; then \ + cargo build --package queryapi_coordinator --offline; \ + else \ + cargo build --release --package queryapi_coordinator --offline; \ + fi FROM ubuntu:20.04 - +ARG CARGO_BUILD_MODE=release RUN apt update && apt install -yy openssl ca-certificates - USER nobody -COPY --from=build /tmp/target/release/queryapi_coordinator /queryapi_coordinator +COPY --from=build /tmp/target/$CARGO_BUILD_MODE/queryapi_coordinator /queryapi_coordinator ENTRYPOINT ["/queryapi_coordinator"] From df380eb5e1b16ef05469ce7f39b8b65a3fc39b43 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 16:53:25 +1200 Subject: [PATCH 4/8] chore: Remove superseeded docker compose file --- indexer/docker-compose.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 indexer/docker-compose.yml diff --git a/indexer/docker-compose.yml b/indexer/docker-compose.yml deleted file mode 100644 index 9be7e5ced..000000000 --- a/indexer/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: "3.9" # optional since v1.27.0 -services: - - actions_queryapi_coordinator: - build: - context: . - args: - - chain_id=mainnet - env_file: ./queryapi_coordinator/.env - links: - - redis - environment: - - REDIS_CONNECTION_STRING=redis://redis - command: - - mainnet - - from-interruption - redis: - image: redis - command: - - redis-server - - "--save 60 1" - - "--loglevel warning" - volumes: - - ./redis/data:/data - ports: - - "6379:6379" \ No newline at end of file From ce5e3241b0ece873e754b7a9466a6e3b3fd111f3 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 16:55:36 +1200 Subject: [PATCH 5/8] fix: Docker compose build --- docker-compose.yml | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 32a167de2..aa4dc242d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,21 @@ version: "3.9" # optional since v1.27.0 services: - actions_queryapi_coordinator: + coordinator: build: context: ./indexer args: - - chain_id=mainnet - links: + - CARGO_BUILD_MODE=debug + depends_on: - redis environment: REDIS_CONNECTION_STRING: redis://redis - LAKE_AWS_SECRET_ACCESS_KEY: - LAKE_AWS_ACCESS_KEY: - QUEUE_AWS_ACCESS_KEY: - QUEUE_AWS_SECRET_ACCESS_KEY: - QUEUE_URL: - START_FROM_BLOCK_QUEUE_URL: + LAKE_AWS_ACCESS_KEY: + LAKE_AWS_SECRET_ACCESS_KEY: + QUEUE_AWS_ACCESS_KEY: + QUEUE_AWS_SECRET_ACCESS_KEY: + QUEUE_URL: MOCK + START_FROM_BLOCK_QUEUE_URL: MOCK PORT: 9180 REGISTRY_CONTRACT_ID: dev-queryapi.dataplatform.near AWS_QUEUE_REGION: eu-central-1 @@ -27,7 +27,7 @@ services: build: context: ./runner depends_on: - - "graphql-engine" + - "hasura-graphql" - "redis" environment: REGION: eu-central-1 @@ -39,8 +39,9 @@ services: PGUSER: postgres PGPASSWORD: postgrespassword PGDATABASE: postgres - AWS_ACCESS_KEY_ID: - AWS_SECRET_ACCESS_KEY: + PORT: 9180 + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: redis: image: redis @@ -49,7 +50,7 @@ services: - "--save 60 1" - "--loglevel warning" volumes: - - ./redis/data:/data + - redis:/data ports: - "6379:6379" @@ -57,7 +58,7 @@ services: image: postgres:12 restart: always volumes: - - db_data:/var/lib/postgresql/data + - postgres:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: postgrespassword ports: @@ -68,8 +69,6 @@ services: context: ./hasura-authentication-service ports: - "4000:4000" - depends_on: - - "hasura-graphql" environment: PORT: 4000 DEFAULT_HASURA_ROLE: append @@ -80,6 +79,7 @@ services: - "8080:8080" depends_on: - "postgres" + - "hasura-auth" restart: always environment: HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres @@ -90,4 +90,5 @@ services: HASURA_GRAPHQL_AUTH_HOOK: http://hasura-auth:4000/auth volumes: - db_data: + postgres: + redis: From 6ed95b61206e95f3439dcef785bd34cf790f52d8 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 17:17:36 +1200 Subject: [PATCH 6/8] chore: Add starting notes to `README` --- README.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ef04df972..9902ba8d6 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ With QueryApi you can * Specify the schema for your own custom hosted database and write to it with your indexer function; * Retrieve that data through a GraphQL API. -# Table of Contents / Applications +## Table of Contents / Applications 1. [QueryApi Coordinator](./indexer) An Indexer that tracks changes to the QueryApi registry contract. It triggers the execution of those IndexerFunctions when they match new blocks by placing messages on an SQS queue. Spawns historical processing threads when needed. @@ -21,3 +21,54 @@ indexer_rules_engine, storage. Stores IndexerFunctions, their schemas and execution parameters like start block height. 6. [Lake Block server](./block-server) Serves blocks from the S3 lake for in browser testing of IndexerFunctions. + +## Getting Started + +The majority of the QueryApi components can be set up locally using Docker. For this purpose, a [Docker Compose file](./docker-compose.yml) has been provided. However, the local system still relies on the NEAR Mainnet, rather than running on a localnet. + +### Requirements +- [Docker](https://docs.docker.com/engine/install/) +- [Docker Compose](https://docs.docker.com/compose/install/) +- [Hasura CLI](https://hasura.io/docs/latest/hasura-cli/install-hasura-cli/) +- AWS Access Keys + +### AWS Credentials Setup +QueryApi requires AWS credentials to stream blocks from [NEAR Lake](https://github.com/near/near-lake-indexer). Credentials are exposed via the following environment variables, which can be found in the Docker Compose file: + +Runner: +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` + +Coordinator: +- `LAKE_AWS_ACCESS_KEY` +- `LAKE_AWS_SECRET_ACCESS_KEY` +- `QUEUE_AWS_ACCESS_KEY` +- `QUEUE_AWS_SECRET_ACCESS_KEY` + +These should be populated with your credentials. In most cases, the same key pair can be used for all 3 sets of credentials. Just ensure the keys have permissions to access S3 for handling [Requestor Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html) in Near Lake. + +### Hasura Configuration +Hasura contains shared tables for e.g. logging and setting arbitrary state. These tables must be configured prior to running the entire QueryApi application. Configuration is stored in the `hasura/` directory and deployed through the Hasura CLI. + +To configure Hasura, first start it with: +```sh +docker compose up hasura-graphql --detach +``` + +And apply the configuration with: +```sh +cd ./hasura && hasura deploy +``` + +### Running QueryApi +With everything configured correctly, we can now start all components of QueryApi with: +```sh +docker compose up +``` + +### Local Configuration +- Coordinator watches the dev registry contract by default (`dev-queryapi.dataplatform.near`). To use a different contract, you can update the `REGISTRY_CONTRACT_ID` environment variable. +- Coodinator will log SQS messages rather than sending them. To use an actual Queue, you can update the `QUEUE_URL` and `START_FROM_BLOCK_QUEUE_URL` environment variables. + +### Initial Provisioning +It is expected to see some errors when starting the QueryAPI Runner for the first time. Before an indexer is executed, it is first provisioned. In the current registry there are most likely many accounts with many indexers under them. On first start up, all Indexers will attempt to provision, and most will fail due to the duplicate attempts to create shared resources. These indexers will eventually retry, and skip provisioning since it has already been setup. From 4d56b883bd5b67a0e2b6689485aff57ab695c7da Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 20:32:36 +1200 Subject: [PATCH 7/8] chore: Add some emojis to readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9902ba8d6..8a19a044c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ With QueryApi you can * Specify the schema for your own custom hosted database and write to it with your indexer function; * Retrieve that data through a GraphQL API. -## Table of Contents / Applications +## 🧩 Components 1. [QueryApi Coordinator](./indexer) An Indexer that tracks changes to the QueryApi registry contract. It triggers the execution of those IndexerFunctions when they match new blocks by placing messages on an SQS queue. Spawns historical processing threads when needed. @@ -22,7 +22,7 @@ indexer_rules_engine, storage. 6. [Lake Block server](./block-server) Serves blocks from the S3 lake for in browser testing of IndexerFunctions. -## Getting Started +## 🚀 Getting Started The majority of the QueryApi components can be set up locally using Docker. For this purpose, a [Docker Compose file](./docker-compose.yml) has been provided. However, the local system still relies on the NEAR Mainnet, rather than running on a localnet. From 016c482977121f362acedf116b7c306a69a788b0 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 9 Aug 2023 20:43:27 +1200 Subject: [PATCH 8/8] chore: Update `Known Issues` section of readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8a19a044c..bb82abb01 100644 --- a/README.md +++ b/README.md @@ -70,5 +70,6 @@ docker compose up - Coordinator watches the dev registry contract by default (`dev-queryapi.dataplatform.near`). To use a different contract, you can update the `REGISTRY_CONTRACT_ID` environment variable. - Coodinator will log SQS messages rather than sending them. To use an actual Queue, you can update the `QUEUE_URL` and `START_FROM_BLOCK_QUEUE_URL` environment variables. -### Initial Provisioning -It is expected to see some errors when starting the QueryAPI Runner for the first time. Before an indexer is executed, it is first provisioned. In the current registry there are most likely many accounts with many indexers under them. On first start up, all Indexers will attempt to provision, and most will fail due to the duplicate attempts to create shared resources. These indexers will eventually retry, and skip provisioning since it has already been setup. +### Known Issues + +It is expected to see some provisioning errors from `Runner` when starting QueryAPI for the first time. These occur when multiple indexers under the same account attempt to provision the same shared infrastructure. These should self resolve after a few seconds.