feat: Support Deployment of Block Streamer (#516)

Block Streamer needs a Dockerfile in order to build the image and be
deployable either locally or in GCP. I've created one for the service
and updated the compose file to set the correct env variables needed.
This also sserves as a record for what ENV variables need to be set
during Terraform deployments to GCP.

In addition, there's a small issue with running QueryApi locally where
if the user runs Runner locally through yarn start, provisioning of
resources through hasura fails because it populates the postgres host as
'localhost' whereas it should be 'postgres', as the calls are processed
inside the hasura docker. This is due to PGHOST being both used and also
passed to Hasura inside Runner. To fix this, I created a separate env
variable called PGHOST_HASURA which can be set to 'postgres'. This way,
Runner can use 'localhost' while it passes 'postgres' to hasura's
docker.

block-streamer/Dockerfile

@@ -0,0 +1,19 @@
+FROM rust:1.75 AS build
+ARG CARGO_BUILD_MODE=release
+WORKDIR /tmp/
+COPY block-streamer/ block-streamer/
+COPY registry/types/ registry/types/
+WORKDIR /tmp/block-streamer/
+RUN apt update && apt install -yy protobuf-compiler
+RUN if [ "$CARGO_BUILD_MODE" = "debug" ]; then \
+        cargo build --package block-streamer; \
+    else \
+        cargo build --release --package block-streamer; \
+    fi
+
+FROM ubuntu:22.04
+ARG CARGO_BUILD_MODE=release
+RUN apt update && apt install -yy openssl ca-certificates
+USER nobody
+COPY --from=build /tmp/block-streamer/target/$CARGO_BUILD_MODE/block-streamer /block-streamer
+ENTRYPOINT ["/block-streamer"]

block-streamer/src/main.rs

@@ -23,12 +23,14 @@ async fn main() -> anyhow::Result<()> {
 
     tracing::info!("Starting Block Streamer Service...");
 
+    tracing::info!("Connecting to Redis...");
     let redis_client = std::sync::Arc::new(redis::RedisClient::connect(&redis_url).await?);
 
     let aws_config = aws_config::from_env().load().await;
     let s3_config = aws_sdk_s3::Config::from(&aws_config);
     let s3_client = crate::s3_client::S3Client::new(s3_config.clone());
 
+    tracing::info!("Connecting to Delta Lake...");
     let delta_lake_client =
         std::sync::Arc::new(crate::delta_lake_client::DeltaLakeClient::new(s3_client));
 

block-streamer/src/server/mod.rs

@@ -10,7 +10,7 @@ pub async fn init(
     delta_lake_client: std::sync::Arc<crate::delta_lake_client::DeltaLakeClient>,
     lake_s3_config: aws_sdk_s3::Config,
 ) -> anyhow::Result<()> {
-    let addr = format!("[::1]:{}", port).parse()?;
+    let addr = format!("0.0.0.0:{}", port).parse()?;
 
     tracing::info!("Starting RPC server at {}", addr);
 

docker-compose.yml

@@ -1,6 +1,24 @@
 version: "3.9"  # optional since v1.27.0
 services:
 
+  block-streamer:
+    build:
+      context: .
+      dockerfile: ./block-streamer/Dockerfile
+      args:
+        - CARGO_BUILD_MODE=debug
+    depends_on:
+      - redis
+    environment:
+      SERVER_PORT: 8002
+      REDIS_URL: redis://redis
+      AWS_ACCESS_KEY_ID:
+      AWS_SECRET_ACCESS_KEY:
+      AWS_REGION: eu-central-1
+      RUST_LOG: info
+    ports:
+      - "8002:8002"
+
   coordinator-v1:
     build:
       context: ./indexer
@@ -12,10 +30,6 @@ services:
       REDIS_CONNECTION_STRING: redis://redis
       LAKE_AWS_ACCESS_KEY:
       LAKE_AWS_SECRET_ACCESS_KEY:
-      QUEUE_AWS_ACCESS_KEY:
-      QUEUE_AWS_SECRET_ACCESS_KEY:
-      QUEUE_URL: MOCK
-      START_FROM_BLOCK_QUEUE_URL: MOCK
       PORT: 9180
       REGISTRY_CONTRACT_ID: dev-queryapi.dataplatform.near
       AWS_QUEUE_REGION: eu-central-1
@@ -50,6 +64,7 @@ services:
       HASURA_ADMIN_SECRET: myadminsecretkey
       REDIS_CONNECTION_STRING: redis://redis
       PGHOST: postgres
+      PGHOST_HASURA: postgres
       PGPORT: 5432
       PGUSER: postgres
       PGPASSWORD: postgrespassword
@@ -58,6 +73,8 @@ services:
       AWS_ACCESS_KEY_ID:
       AWS_SECRET_ACCESS_KEY:
       GRPC_SERVER_PORT: 7001
+    ports:
+      - "7001:7001"
 
   redis:
     image: redis

runner/src/hasura-client/hasura-client.test.ts

@@ -8,6 +8,7 @@ describe('HasuraClient', () => {
   const HASURA_ENDPOINT = 'mock-hasura-endpoint';
   const HASURA_ADMIN_SECRET = 'mock-hasura-admin-secret';
   const PGHOST = 'localhost';
+  const PGHOST_HASURA = 'localhost';
   const PGPORT = '5432';
 
   beforeAll(() => {
@@ -16,6 +17,7 @@ describe('HasuraClient', () => {
       HASURA_ENDPOINT,
       HASURA_ADMIN_SECRET,
       PGHOST,
+      PGHOST_HASURA,
       PGPORT,
     };
   });

runner/src/hasura-client/hasura-client.ts

@@ -337,7 +337,7 @@ export default class HasuraClient {
               password,
               database: databaseName,
               username: userName,
-              host: process.env.PGHOST,
+              host: process.env.PGHOST_HASURA ?? process.env.PGHOST,
               port: Number(process.env.PGPORT),
             }
           },

runner/src/server/runner-server.ts

@@ -20,13 +20,13 @@ export default function startRunnerServer (executors: Map<string, StreamHandler>
   assert(process.env.GRPC_SERVER_PORT, 'GRPC_SERVER_PORT is not defined');
 
   server.bindAsync(
-    `localhost:${process.env.GRPC_SERVER_PORT}`,
+    `0.0.0.0:${process.env.GRPC_SERVER_PORT}`,
     credentials.createInsecure(), // TODO: Use secure credentials with allow for Coordinator
     (err: Error | null, port: number) => {
       if (err) {
         console.error(`Server error: ${err.message}`);
       } else {
-        console.log(`gRPC server bound on port: ${port}`);
+        console.log(`gRPC server bound on: 0.0.0.0:${port}`);
         server.start();
       }
     }