From c8c2af2fb88eb8aa3b6ec4724fb902d9c1a665d8 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Wed, 17 Jan 2024 17:34:53 -0800 Subject: [PATCH 1/7] feat: Add Dockerfile and Update Compose File to Run Block Streamer --- block-streamer/Dockerfile | 16 ++++++++++++++++ block-streamer/src/main.rs | 2 ++ block-streamer/src/server/mod.rs | 2 +- docker-compose.yml | 22 ++++++++++++++++++---- 4 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 block-streamer/Dockerfile diff --git a/block-streamer/Dockerfile b/block-streamer/Dockerfile new file mode 100644 index 000000000..1eba99a86 --- /dev/null +++ b/block-streamer/Dockerfile @@ -0,0 +1,16 @@ +FROM rust:1.75 AS build +ARG CARGO_BUILD_MODE=debug +WORKDIR /tmp/ +COPY block-streamer/ block-streamer/ +COPY registry/types/ registry/types/ +WORKDIR /tmp/block-streamer/ +RUN apt update && apt install -yy protobuf-compiler +RUN cargo build --package block-streamer + +FROM ubuntu:22.04 +ARG CARGO_BUILD_MODE=debug +ENV RUST_LOG=info +RUN apt update && apt install -yy openssl ca-certificates +USER nobody +COPY --from=build /tmp/block-streamer/target/$CARGO_BUILD_MODE/block-streamer /block-streamer +ENTRYPOINT ["/block-streamer"] diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 39d114a55..fdb3e0a40 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -25,6 +25,8 @@ async fn main() -> anyhow::Result<()> { let redis_client = std::sync::Arc::new(redis::RedisClient::connect(&redis_url).await?); + tracing::info!("Connected to Redis"); + let aws_config = aws_config::from_env().load().await; let s3_config = aws_sdk_s3::Config::from(&aws_config); let s3_client = crate::s3_client::S3Client::new(s3_config.clone()); diff --git a/block-streamer/src/server/mod.rs b/block-streamer/src/server/mod.rs index f72dc3c1d..a83535dda 100644 --- a/block-streamer/src/server/mod.rs +++ b/block-streamer/src/server/mod.rs @@ -10,7 +10,7 @@ pub async fn init( delta_lake_client: std::sync::Arc, lake_s3_config: aws_sdk_s3::Config, ) -> anyhow::Result<()> { - let addr = format!("[::1]:{}", port).parse()?; + let addr = format!("[::]:{}", port).parse()?; tracing::info!("Starting RPC server at {}", addr); diff --git a/docker-compose.yml b/docker-compose.yml index 71f07be71..0cbf1777d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,24 @@ version: "3.9" # optional since v1.27.0 services: + + block-streamer: + build: + context: . + dockerfile: ./block-streamer/Dockerfile + args: + - CARGO_BUILD_MODE=debug + depends_on: + - redis + environment: + SERVER_PORT: 8002 + REDIS_URL: redis://redis + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: eu-central-1 + ports: + - "8002:8002" + coordinator-v1: build: context: ./indexer @@ -12,10 +30,6 @@ services: REDIS_CONNECTION_STRING: redis://redis LAKE_AWS_ACCESS_KEY: LAKE_AWS_SECRET_ACCESS_KEY: - QUEUE_AWS_ACCESS_KEY: - QUEUE_AWS_SECRET_ACCESS_KEY: - QUEUE_URL: MOCK - START_FROM_BLOCK_QUEUE_URL: MOCK PORT: 9180 REGISTRY_CONTRACT_ID: dev-queryapi.dataplatform.near AWS_QUEUE_REGION: eu-central-1 From ff396f63547234eb273d4016f68c10704a7f109c Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 18 Jan 2024 14:30:46 -0800 Subject: [PATCH 2/7] fix: Make Hasura PG Host different for local dev without code change --- docker-compose.yml | 1 + runner/src/hasura-client/hasura-client.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0cbf1777d..18d91d9b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,6 +64,7 @@ services: HASURA_ADMIN_SECRET: myadminsecretkey REDIS_CONNECTION_STRING: redis://redis PGHOST: postgres + PGHOST_HASURA: postgres PGPORT: 5432 PGUSER: postgres PGPASSWORD: postgrespassword diff --git a/runner/src/hasura-client/hasura-client.ts b/runner/src/hasura-client/hasura-client.ts index 0162c4b41..320969f7a 100644 --- a/runner/src/hasura-client/hasura-client.ts +++ b/runner/src/hasura-client/hasura-client.ts @@ -337,7 +337,7 @@ export default class HasuraClient { password, database: databaseName, username: userName, - host: process.env.PGHOST, + host: process.env.PGHOST_HASURA, port: Number(process.env.PGPORT), } }, From fc06809cf9b7ec864dd0a7a610b18add7a841aed Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 18 Jan 2024 17:21:58 -0800 Subject: [PATCH 3/7] fix: Fix Failing Hasura Test --- runner/src/hasura-client/hasura-client.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runner/src/hasura-client/hasura-client.test.ts b/runner/src/hasura-client/hasura-client.test.ts index d4b621886..2787af9f8 100644 --- a/runner/src/hasura-client/hasura-client.test.ts +++ b/runner/src/hasura-client/hasura-client.test.ts @@ -8,6 +8,7 @@ describe('HasuraClient', () => { const HASURA_ENDPOINT = 'mock-hasura-endpoint'; const HASURA_ADMIN_SECRET = 'mock-hasura-admin-secret'; const PGHOST = 'localhost'; + const PGHOST_HASURA = 'localhost'; const PGPORT = '5432'; beforeAll(() => { @@ -16,6 +17,7 @@ describe('HasuraClient', () => { HASURA_ENDPOINT, HASURA_ADMIN_SECRET, PGHOST, + PGHOST_HASURA, PGPORT, }; }); From a5ca959666a7f2646d0f3f2ccf45a0d9792efb25 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Fri, 19 Jan 2024 13:16:58 -0800 Subject: [PATCH 4/7] fix: Some small logging and PGHOST fallback if PGHOST_HASURA is unset --- block-streamer/Dockerfile | 6 +++++- block-streamer/src/main.rs | 5 +++-- runner/src/hasura-client/hasura-client.ts | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/block-streamer/Dockerfile b/block-streamer/Dockerfile index 1eba99a86..6938c6793 100644 --- a/block-streamer/Dockerfile +++ b/block-streamer/Dockerfile @@ -5,7 +5,11 @@ COPY block-streamer/ block-streamer/ COPY registry/types/ registry/types/ WORKDIR /tmp/block-streamer/ RUN apt update && apt install -yy protobuf-compiler -RUN cargo build --package block-streamer +RUN if [ "$CARGO_BUILD_MODE" = "debug" ]; then \ + cargo build --package block-streamer; \ + else \ + cargo build --release --package block-streamer; \ + fi FROM ubuntu:22.04 ARG CARGO_BUILD_MODE=debug diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index fdb3e0a40..0a30fc189 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -23,17 +23,18 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Starting Block Streamer Service..."); + tracing::info!("Connecting to Redis..."); let redis_client = std::sync::Arc::new(redis::RedisClient::connect(&redis_url).await?); - tracing::info!("Connected to Redis"); - let aws_config = aws_config::from_env().load().await; let s3_config = aws_sdk_s3::Config::from(&aws_config); let s3_client = crate::s3_client::S3Client::new(s3_config.clone()); + tracing::info!("Connecting to Delta Lake..."); let delta_lake_client = std::sync::Arc::new(crate::delta_lake_client::DeltaLakeClient::new(s3_client)); + tracing::info!("Starting gRPC Server..."); server::init(&server_port, redis_client, delta_lake_client, s3_config).await?; Ok(()) diff --git a/runner/src/hasura-client/hasura-client.ts b/runner/src/hasura-client/hasura-client.ts index 320969f7a..3bf4de72a 100644 --- a/runner/src/hasura-client/hasura-client.ts +++ b/runner/src/hasura-client/hasura-client.ts @@ -337,7 +337,7 @@ export default class HasuraClient { password, database: databaseName, username: userName, - host: process.env.PGHOST_HASURA, + host: process.env.PGHOST_HASURA ?? process.env.PGHOST, port: Number(process.env.PGPORT), } }, From 9fb9e75cce9d411b4a1e448c480e802e2cf18114 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Mon, 22 Jan 2024 13:07:02 -0800 Subject: [PATCH 5/7] fix: Change server addresses, set build to release, clean up --- block-streamer/Dockerfile | 4 ++-- block-streamer/src/main.rs | 1 - docker-compose.yml | 2 ++ runner/src/server/runner-server.ts | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/block-streamer/Dockerfile b/block-streamer/Dockerfile index 6938c6793..c6f9607da 100644 --- a/block-streamer/Dockerfile +++ b/block-streamer/Dockerfile @@ -1,5 +1,5 @@ FROM rust:1.75 AS build -ARG CARGO_BUILD_MODE=debug +ARG CARGO_BUILD_MODE=release WORKDIR /tmp/ COPY block-streamer/ block-streamer/ COPY registry/types/ registry/types/ @@ -12,7 +12,7 @@ RUN if [ "$CARGO_BUILD_MODE" = "debug" ]; then \ fi FROM ubuntu:22.04 -ARG CARGO_BUILD_MODE=debug +ARG CARGO_BUILD_MODE=release ENV RUST_LOG=info RUN apt update && apt install -yy openssl ca-certificates USER nobody diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 0a30fc189..1d5a3ac2c 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -34,7 +34,6 @@ async fn main() -> anyhow::Result<()> { let delta_lake_client = std::sync::Arc::new(crate::delta_lake_client::DeltaLakeClient::new(s3_client)); - tracing::info!("Starting gRPC Server..."); server::init(&server_port, redis_client, delta_lake_client, s3_config).await?; Ok(()) diff --git a/docker-compose.yml b/docker-compose.yml index 18d91d9b3..345488d2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,8 @@ services: AWS_ACCESS_KEY_ID: AWS_SECRET_ACCESS_KEY: GRPC_SERVER_PORT: 7001 + ports: + - "7001:7001" redis: image: redis diff --git a/runner/src/server/runner-server.ts b/runner/src/server/runner-server.ts index 64b1d4652..efe3283e3 100644 --- a/runner/src/server/runner-server.ts +++ b/runner/src/server/runner-server.ts @@ -20,7 +20,7 @@ export default function startRunnerServer (executors: Map assert(process.env.GRPC_SERVER_PORT, 'GRPC_SERVER_PORT is not defined'); server.bindAsync( - `localhost:${process.env.GRPC_SERVER_PORT}`, + `0.0.0.0:${process.env.GRPC_SERVER_PORT}`, credentials.createInsecure(), // TODO: Use secure credentials with allow for Coordinator (err: Error | null, port: number) => { if (err) { From d8f67042f97be1c8000c3d56cef623577dfb89a4 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Tue, 23 Jan 2024 16:32:02 -0800 Subject: [PATCH 6/7] Change Runner endpoint, migrated RUST_LOG to compose file --- block-streamer/Dockerfile | 1 - docker-compose.yml | 1 + runner/src/server/runner-server.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/block-streamer/Dockerfile b/block-streamer/Dockerfile index c6f9607da..e106c9bd1 100644 --- a/block-streamer/Dockerfile +++ b/block-streamer/Dockerfile @@ -13,7 +13,6 @@ RUN if [ "$CARGO_BUILD_MODE" = "debug" ]; then \ FROM ubuntu:22.04 ARG CARGO_BUILD_MODE=release -ENV RUST_LOG=info RUN apt update && apt install -yy openssl ca-certificates USER nobody COPY --from=build /tmp/block-streamer/target/$CARGO_BUILD_MODE/block-streamer /block-streamer diff --git a/docker-compose.yml b/docker-compose.yml index 345488d2e..ba067adb2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: AWS_ACCESS_KEY_ID: AWS_SECRET_ACCESS_KEY: AWS_REGION: eu-central-1 + RUST_LOG: info ports: - "8002:8002" diff --git a/runner/src/server/runner-server.ts b/runner/src/server/runner-server.ts index efe3283e3..32a7e3f8d 100644 --- a/runner/src/server/runner-server.ts +++ b/runner/src/server/runner-server.ts @@ -20,7 +20,7 @@ export default function startRunnerServer (executors: Map assert(process.env.GRPC_SERVER_PORT, 'GRPC_SERVER_PORT is not defined'); server.bindAsync( - `0.0.0.0:${process.env.GRPC_SERVER_PORT}`, + `[::]:${process.env.GRPC_SERVER_PORT}`, credentials.createInsecure(), // TODO: Use secure credentials with allow for Coordinator (err: Error | null, port: number) => { if (err) { From 2ac0a865cafbb44ea52914c3a0a616f24a57c284 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Tue, 23 Jan 2024 17:21:04 -0800 Subject: [PATCH 7/7] fix: Set both grpc servers to run on 0.0.0.0 --- block-streamer/src/server/mod.rs | 2 +- docker-compose.yml | 1 - runner/src/server/runner-server.ts | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/block-streamer/src/server/mod.rs b/block-streamer/src/server/mod.rs index a83535dda..10099b3f3 100644 --- a/block-streamer/src/server/mod.rs +++ b/block-streamer/src/server/mod.rs @@ -10,7 +10,7 @@ pub async fn init( delta_lake_client: std::sync::Arc, lake_s3_config: aws_sdk_s3::Config, ) -> anyhow::Result<()> { - let addr = format!("[::]:{}", port).parse()?; + let addr = format!("0.0.0.0:{}", port).parse()?; tracing::info!("Starting RPC server at {}", addr); diff --git a/docker-compose.yml b/docker-compose.yml index ba067adb2..48103d4ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,6 @@ version: "3.9" # optional since v1.27.0 services: - block-streamer: build: context: . diff --git a/runner/src/server/runner-server.ts b/runner/src/server/runner-server.ts index 32a7e3f8d..fe1dcce96 100644 --- a/runner/src/server/runner-server.ts +++ b/runner/src/server/runner-server.ts @@ -20,13 +20,13 @@ export default function startRunnerServer (executors: Map assert(process.env.GRPC_SERVER_PORT, 'GRPC_SERVER_PORT is not defined'); server.bindAsync( - `[::]:${process.env.GRPC_SERVER_PORT}`, + `0.0.0.0:${process.env.GRPC_SERVER_PORT}`, credentials.createInsecure(), // TODO: Use secure credentials with allow for Coordinator (err: Error | null, port: number) => { if (err) { console.error(`Server error: ${err.message}`); } else { - console.log(`gRPC server bound on port: ${port}`); + console.log(`gRPC server bound on: 0.0.0.0:${port}`); server.start(); } }