From 5789cea34342d9bbece35b00c67fac98b699c8eb Mon Sep 17 00:00:00 2001 From: Tsotne Nazarashvili Date: Sun, 25 Feb 2024 12:10:03 +0400 Subject: [PATCH] Update configuration and Docker files --- .env.example | 2 + .github/workflows/docker-indexer.yml | 53 +++++++++++ .../workflows/{docker.yml => docker-web.yml} | 3 +- Dockerfile => Dockerfile.indexer | 7 +- Dockerfile.web | 30 +++++++ README.md | 90 +++++++++++++------ apps/indexer/src/env.ts | 3 +- apps/indexer/src/events.ts | 6 +- apps/indexer/src/index.ts | 6 +- apps/web/app/env.server.ts | 5 +- apps/web/package.json | 3 + docker-compose.yml | 17 +++- package.json | 7 -- pnpm-lock.yaml | 2 +- 14 files changed, 186 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/docker-indexer.yml rename .github/workflows/{docker.yml => docker-web.yml} (93%) rename Dockerfile => Dockerfile.indexer (76%) create mode 100644 Dockerfile.web diff --git a/.env.example b/.env.example index 6c75d6b..3b205bc 100644 --- a/.env.example +++ b/.env.example @@ -13,3 +13,5 @@ INDEXER_CONFIG=" # Mainnet contract SN_MAIN 4982 0x6a05844a03bb9e744479e3298f54705a35966ab04140d3d8dd797c1f6dc49d0 " +INDEXER_DELAY=3000 +LOG_LEVEL=info diff --git a/.github/workflows/docker-indexer.yml b/.github/workflows/docker-indexer.yml new file mode 100644 index 0000000..a88c7e0 --- /dev/null +++ b/.github/workflows/docker-indexer.yml @@ -0,0 +1,53 @@ +name: Docker image + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "main" + - "docker" + tags: + - "v*" + pull_request: + branches: + - "dev" + +jobs: + docker: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/thecotne/gol2-indexer + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile.indexer + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker-web.yml similarity index 93% rename from .github/workflows/docker.yml rename to .github/workflows/docker-web.yml index 3fc2289..a256e4c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker-web.yml @@ -33,7 +33,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository }} + images: ghcr.io/thecotne/gol2-web - name: Log in to the Container registry uses: docker/login-action@v3 @@ -46,6 +46,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: Dockerfile.web push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile.indexer similarity index 76% rename from Dockerfile rename to Dockerfile.indexer index fb34432..650e40f 100644 --- a/Dockerfile +++ b/Dockerfile.indexer @@ -3,22 +3,19 @@ FROM node:20.11.0-alpine as base FROM base as src WORKDIR /src -ENV PNPM_HOME="/pnpm" -ENV PATH="$PNPM_HOME:$PATH" RUN corepack enable COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ COPY apps/indexer/package.json ./apps/indexer/ -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN pnpm install --frozen-lockfile COPY apps/indexer/src ./apps/indexer/src/ COPY apps/indexer/tsconfig.json ./apps/indexer/ -RUN pnpm build +RUN pnpm run --filter=./apps/indexer build RUN pnpm deploy --filter=./apps/indexer --prod /app -# build final image FROM base as app WORKDIR /app diff --git a/Dockerfile.web b/Dockerfile.web new file mode 100644 index 0000000..644ebd4 --- /dev/null +++ b/Dockerfile.web @@ -0,0 +1,30 @@ +FROM node:20.11.0-alpine as base + +FROM base as src +WORKDIR /src + +RUN corepack enable + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY apps/web/package.json ./apps/web/ + +RUN pnpm install --frozen-lockfile + +COPY apps/web/public ./apps/web/public/ +COPY apps/web/app ./apps/web/app/ +COPY apps/web/postcss.config.js ./apps/web/ +COPY apps/web/tailwind.config.js ./apps/web/ +COPY apps/web/tsconfig.json ./apps/web/ +COPY apps/web/vite.config.ts ./apps/web/ + +RUN pnpm run --filter=./apps/web build +RUN pnpm deploy --filter=./apps/web --prod /app + +FROM base as app +WORKDIR /app + +COPY --from=src /app ./ + +EXPOSE 3000 + +CMD ["./node_modules/.bin/remix-serve ./build/server/index.js"] diff --git a/README.md b/README.md index 8549120..f23e0a1 100644 --- a/README.md +++ b/README.md @@ -5,39 +5,70 @@ - `source ./setup-install.sh` - install nvm, node and pnpm. - `source ./setup-update.sh` - install up-to-date versions of nvm, node and pnpm. - `pnpm dbmate up` - create the database and run migrations +- `pnpm -r build` - build all the projects +- `pnpm -r kanel` - generate types from the database for all the projects +- `pnpm -r dev` - start all the projects +- `pnpm -r debug` - start all the projects with debug logs +- `pnpm -r clean` - remove `dist` for all the projects + +### Docker Compose commands + +- `docker compose build` - build the project with docker + - `docker compose build indexer` - build the indexer with docker + - `docker compose build web` - build the web with docker +- `docker compose up` - start the project with docker + - `docker compose up -d` - start the project with docker in detached mode + - `docker compose up db -d` - start the database with docker +- `dpcker compose rm` - remove the project with docker + - `docker compose rm --stop` - stop and remove the project with docker + +### Scripts defined in `package.json` + - `pnpm pgcli` - connect to the database with pgcli -- `pnpm build` - build the project -- `pnpm kanel` - generate types from the database -- `pnpm dev` - start the project -- `pnpm debug` - start the project with debug logs -- `pnpm clean` - remove `dist` - `pnpm check` - check the code - `pnpm fix` - check and fromat the code -- `docker compose up --build` - start the project with docker -- `docker compose up -d` - start the project with docker in detached mode -- `docker compose up db -d` - start the database with docker -- `docker compose rm --stop` - stop and remove the project with docker + +### Scripts defined in `apps/indexer/package.json` + +- `pnpm -F ./apps/indexer kanel` - generate types from the database +- `pnpm -F ./apps/indexer build` - build the project +- `pnpm -F ./apps/indexer dev` - start the project +- `pnpm -F ./apps/indexer debug` - start the project with debug logs +- `pnpm -F ./apps/indexer test` - check the code +- `pnpm -F ./apps/indexer clean` - remove `dist` + +### Scripts defined in `apps/web/package.json` + +- `pnpm -F ./apps/web kanel` - generate types from the database +- `pnpm -F ./apps/web build` - build the project +- `pnpm -F ./apps/web dev` - start the project +- `pnpm -F ./apps/web lint` - check the code +- `pnpm -F ./apps/web start` - start the project +- `pnpm -F ./apps/web test` - check the code ## Env variables -- `DBMATE_MIGRATIONS_DIR` - The directory where the migrations are located -- `DBMATE_SCHEMA_FILE` - The file where the schema is located -- `INFURA_API_KEY` - The Infura API key +### Required Env Variables + - `DATABASE_URL` - Database connection string -- `CONTRACT_ADDRESS` - The contract address - - [0x06dc4bd1212e67fd05b456a34b24a060c45aad08ab95843c42af31f86c7bd093](https://testnet.starkscan.co/contract/0x06dc4bd1212e67fd05b456a34b24a060c45aad08ab95843c42af31f86c7bd093) for old contract on Goerli - - [0x077c08aed31ac023981de21e60b0bc0958a05f81943be057919c628994de86fe](https://testnet.starkscan.co/contract/0x077c08aed31ac023981de21e60b0bc0958a05f81943be057919c628994de86fe) for new contract on Goerli - - [0x06a05844a03bb9e744479e3298f54705a35966ab04140d3d8dd797c1f6dc49d0](https://starkscan.co/contract/0x06a05844a03bb9e744479e3298f54705a35966ab04140d3d8dd797c1f6dc49d0) for Mainnet - - 🤷‍♂️ for Sepolia -- `STARKNET_NETWORK_NAME` - The StarkNet network name - - `SN_MAIN` for Mainnet - - `SN_GOERLI` for Goerli - - `SN_SEPOLIA` for Sepolia -- `CONTRACT_BLOCK_NUMBER` - The block number at which the contract was deployed - - `267275` for old contract on Goerli - - `921928` for new contract on Goerli - - `4982` for Mainnet - - 🤷‍♂️ for Sepolia +- `INDEXER_CONFIG` - The indexer configuration + - `STARKNET_NETWORK_NAME CONTRACT_BLOCK_NUMBER CONTRACT_ADDRESS` for each contract + - `STARKNET_NETWORK_NAME` - The StarkNet network name (`SN_GOERLI` or `SN_MAIN`) + - `CONTRACT_BLOCK_NUMBER` - The block number at which the contract was deployed + - `CONTRACT_ADDRESS` - The contract address + - Example: + - `SN_GOERLI 942318 0x5bd17bba6b3cb9740bcc0f20f93ecf443250c4f09d94e4ab32d3bdffc7ebba2` + - `SN_MAIN 526494 0x79294c688eb80e025f298b1ab2d30dd7a4a316ed592ac2fc124710564e4e911` + - `SN_GOERLI 267275 0x06dc4bd1212e67fd05b456a34b24a060c45aad08ab95843c42af31f86c7bd093` + - `SN_MAIN 4982 0x6a05844a03bb9e744479e3298f54705a35966ab04140d3d8dd797c1f6dc49d0` + +### Optional Env Variables + +- `DBMATE_MIGRATIONS_DIR` - The directory where the migrations are located (default: `db/migrations`) +- `DBMATE_SCHEMA_FILE` - The file where the schema is located (default: `db/schema.sql`) +- `INFURA_API_KEY` - The Infura API key (nullable) +- `INDEXER_DELAY` - The delay between indexing (in milliseconds) (default: `3000`) +- `LOG_LEVEL` - The log level (`trace`, `debug`, `info`, `warn`, `error`, `fatal`) (default: `info`) ## Tools and libraries used @@ -63,6 +94,13 @@ - [winston](https://github.com/winstonjs/winston) - A logger for just about everything. - [znv](https://github.com/lostfictions/znv) - Type-safe environment parsing and validation for Node.js with Zod schemas. - [Zod](https://zod.dev/) - TypeScript-first schema validation with static type inference. +- [react](https://react.dev/) - A JavaScript library for building user interfaces. +- [remix-run](https://remix.run/) - The full-stack web framework for the modern web. +- [Vite](https://vitejs.dev/) - Next generation frontend tooling. It's fast! +- [Tailwind CSS](https://tailwindcss.com/) - A utility-first CSS framework for rapid UI development. +- [PostCSS](https://postcss.org/) - A tool for transforming CSS with JavaScript plugins. +- [tanstack/react-table](https://tanstack.com/table/latest) - Headless UI for building powerful tables & datagrids. +- [shadcn-ui](https://ui.shadcn.com/) - Beautifully designed components that you can copy and paste into your apps. ## Related projects diff --git a/apps/indexer/src/env.ts b/apps/indexer/src/env.ts index c509c4e..a8aa5e9 100644 --- a/apps/indexer/src/env.ts +++ b/apps/indexer/src/env.ts @@ -15,8 +15,9 @@ if (process.env.NODE_ENV !== "production") { export const env = parseEnv(process.env, { DATABASE_URL: z.string(), - INFURA_API_KEY: z.string().nullable(), INDEXER_CONFIG: z.string(), + INDEXER_DELAY: z.number().default(3000), + INFURA_API_KEY: z.string().nullable(), LOG_LEVEL: z .enum(["trace", "debug", "info", "warn", "error", "fatal"]) .default("info"), diff --git a/apps/indexer/src/events.ts b/apps/indexer/src/events.ts index 9c44a0d..92611ba 100644 --- a/apps/indexer/src/events.ts +++ b/apps/indexer/src/events.ts @@ -11,7 +11,7 @@ import { num, uint256, } from "starknet"; -import { db, log, starknet, starknetMainnet } from "./env"; +import { db, env, log, starknet, starknetMainnet } from "./env"; import { EventContractAddress, EventEventIndex, @@ -364,7 +364,9 @@ export async function pullEvents( log.info("Inserted events.", { events: eventsChunk.events.length }); - // await new Promise((resolve) => setTimeout(resolve, 3000)); + if (env.INDEXER_DELAY > 0) { + await new Promise((resolve) => setTimeout(resolve, env.INDEXER_DELAY)); + } } } while (eventsChunk.continuation_token); } diff --git a/apps/indexer/src/index.ts b/apps/indexer/src/index.ts index 69f0a0c..7022742 100644 --- a/apps/indexer/src/index.ts +++ b/apps/indexer/src/index.ts @@ -1,4 +1,4 @@ -import { contracts, log } from "./env"; +import { contracts, env, log } from "./env"; import { pullEvents } from "./events"; void main(); @@ -20,7 +20,9 @@ async function main() { await pullEvents(contractAddress, blockNumber, networkName); } - // await new Promise((resolve) => setTimeout(resolve, 3000)); + if (env.INDEXER_DELAY > 0) { + await new Promise((resolve) => setTimeout(resolve, env.INDEXER_DELAY)); + } } } catch (e) { log.error("App failed.", e); diff --git a/apps/web/app/env.server.ts b/apps/web/app/env.server.ts index c509c4e..da51ed2 100644 --- a/apps/web/app/env.server.ts +++ b/apps/web/app/env.server.ts @@ -6,7 +6,7 @@ import { parseEnv } from "znv"; import { z } from "zod"; import PublicSchema from "./schemas/Database"; -if (process.env.NODE_ENV !== "production") { +if (import.meta.env.DEV) { const dotenv = await import("dotenv"); const path = await import("path"); @@ -15,8 +15,9 @@ if (process.env.NODE_ENV !== "production") { export const env = parseEnv(process.env, { DATABASE_URL: z.string(), - INFURA_API_KEY: z.string().nullable(), INDEXER_CONFIG: z.string(), + INDEXER_DELAY: z.number().default(3000), + INFURA_API_KEY: z.string().nullable(), LOG_LEVEL: z .enum(["trace", "debug", "info", "warn", "error", "fatal"]) .default("info"), diff --git a/apps/web/package.json b/apps/web/package.json index 6f9190a..46e0dbb 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -3,6 +3,9 @@ "private": true, "sideEffects": false, "type": "module", + "files": [ + "build" + ], "scripts": { "kanel": "kanel -c ./.kanelrc.cjs", "build": "remix vite:build", diff --git a/docker-compose.yml b/docker-compose.yml index 2a99184..acc5b91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,9 +12,24 @@ services: ports: - "5432:5432" indexer: - build: . + image: "gol2-indexer" + build: + context: . + dockerfile: Dockerfile.indexer depends_on: - db env_file: - .env - .env.docker + web: + image: "gol2-web" + build: + context: . + dockerfile: Dockerfile.web + depends_on: + - db + env_file: + - .env + - .env.docker + ports: + - "3000:3000" diff --git a/package.json b/package.json index 8841590..8d6c50f 100644 --- a/package.json +++ b/package.json @@ -6,13 +6,6 @@ "license": "UNLICENSED", "files": [], "scripts": { - "build": "pnpm -r build", - "test": "pnpm -r test", - "lint": "pnpm -r lint", - "dev": "pnpm -r dev", - "debug": "pnpm -r debug", - "clean": "pnpm -r clean", - "kanel": "pnpm -r kanel", "pgcli": "dotenv -- pnpm -c exec 'pgcli \"$DATABASE_URL\"'", "check": "biome check .", "fix": "biome check . --apply" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3c7a17f..e1bd2cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -187,7 +187,7 @@ importers: specifier: ^8.0.0 version: 8.0.0(react@18.3.0-canary-98b8359f6-20240223) isbot: - specifier: ^4 + specifier: ^4.4.0 version: 4.4.0 kysely: specifier: ^0.27.2