Merge pull request #2966 from w3f/staging

Deploying changes from staging

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2.1 
+version: 2.1
 
 jobs:
   checkDependencies:
@@ -12,7 +12,7 @@ jobs:
 
   checkCandidatesFile:
     docker:
-      - image: node:18
+      - image: node:22
     resource_class: large
     steps:
       - checkout
@@ -24,7 +24,7 @@ jobs:
 
   checkCoreESLint:
     docker:
-      - image: node:18
+      - image: node:22
     resource_class: large
     steps:
       - checkout
@@ -36,7 +36,7 @@ jobs:
 
   checkCommonESLint:
     docker:
-      - image: node:18
+      - image: node:22
     resource_class: large
     steps:
       - checkout
@@ -49,7 +49,7 @@ jobs:
   CommonUnitTests:
     description: "Common Unit Tests"
     docker:
-      - image: node:18-bullseye
+      - image: node:22-bullseye
       - image: mongo:6.0.9
         name: mongodb
     resource_class: xlarge
@@ -67,7 +67,7 @@ jobs:
   ChaindataIntegrationTests:
     description: "Chaindata Integration Tests"
     docker:
-      - image: node:18-bullseye
+      - image: node:22-bullseye
       - image: mongo:6.0.9
         name: mongodb
     resource_class: xlarge
@@ -85,7 +85,7 @@ jobs:
   ApiHandlerIntegrationTests:
     description: "ApiHandler Integration Tests"
     docker:
-      - image: node:18-bullseye
+      - image: node:22-bullseye
       - image: mongo:6.0.9
         name: mongodb
     resource_class: xlarge
@@ -103,7 +103,7 @@ jobs:
   NominatorIntegrationTests:
     description: "Nominator Integration Tests"
     docker:
-      - image: node:18-bullseye
+      - image: node:22-bullseye
       - image: mongo:6.0.9
         name: mongodb
     resource_class: xlarge
@@ -121,7 +121,7 @@ jobs:
   ScorekeeperIntegrationTests:
     description: "Scorekeeper Integration Tests"
     docker:
-      - image: node:18-bullseye
+      - image: node:22-bullseye
       - image: mongo:6.0.9
         name: mongodb
 
@@ -140,7 +140,7 @@ jobs:
   TelemetryIntegrationTests:
     description: "Telemetry Integration Tests"
     docker:
-      - image: node:18-bullseye
+      - image: node:22-bullseye
       - image: mongo:6.0.9
         name: mongodb
 

.gitignore

@@ -28,8 +28,6 @@ dist
 **/config/main.json
 **/config/secret.json
 
-redis/
-
 docs/.DS_Store
 /packages/core/kusama-matrix.txt
 /packages/core/polkadot-matrix.txt
@@ -43,4 +41,4 @@ docs/.DS_Store
 build/**
 dist/**
 .next/**
-coverage/**
+coverage/**

.yarnrc.yml

@@ -11,8 +11,3 @@ plugins:
 pnpMode: loose
 
 yarnPath: .yarn/releases/yarn-3.2.2.cjs
-
-packageExtensions:
-  esbuild@*:
-    dependencies:
-      "@esbuild/linux-arm64": "*"

Dockerfile

@@ -1,17 +1,23 @@
-FROM node:21-slim AS builder
-ARG MATRIX_TOKEN
-ARG PACKAGE
-ENV PACKAGE ${PACKAGE}
+FROM node:22-slim AS docs_builder
 WORKDIR /code
 
 COPY docs docs
 RUN cd docs && npm install && npm run build
 
+FROM node:22-slim
+WORKDIR /code
+ARG MATRIX_TOKEN
+ARG PACKAGE
+ENV PACKAGE ${PACKAGE}
+
+COPY --from=docs_builder /code/docs/build/ /code/docs/build/
+
 COPY . .
 
 RUN --mount=type=cache,target=/code/.yarn/cache \
     --mount=type=cache,target=/turbo_cache \
     yarn install --immutable && \
-    yarn turbo --cache-dir /turbo_cache
+    yarn turbo --cache-dir /turbo_cache && \
+    yarn workspaces focus --production
 
 CMD yarn run start:js:${PACKAGE}

README.md

@@ -26,52 +26,30 @@ The nominating backend will routinely change its nominations at every era. The b
 
 > A monorepo containing TypeScript microservices for the Thousand Validators Program.
 
-The following is a monorepo of packages for the Thousand Validators Program. Each package is a microservice that can be run independently or together with other microservices. 
+The following is a monorepo of packages for the Thousand Validators Program.  
 
 The monorepo is managed using Yarn workspaces, and contains the following packages:
 - [`packages/common`](packages/common): A package containing common code shared across all microservices.
 - [`packages/core`](packages/core): A package containing the core logic of the Thousand Validators Program.
 - [`packages/gateway`](packages/gateway): A package for an API gateway that exposes the backend with a REST API.
 - [`packages/telemetry`](packages/telemetry): A package for a telemetry client that monitors uptime
-- [`packages/worker`](packages/worker): A packages for job queue workers that perform background tasks.
-
-
 
 ## Installation & Setup
 
 ### Instances
 
 There's a few ways of running the backend with docker containers, either in kubernetes, or with docker-compose.
 
-There is the `Current / Monolith` way of running instances, and the `Microservice` way of running instances.
-
-`Current / Monolith` Architecture:
-
-![Current / Monolith Architecture](./docs/architecture/monolith.png)
+Current architecture:
 
-
-`Microservice` Architecture:
-
-![Microservice Architecture](./docs/architecture/microservice.png)
+![Current Architecture](./docs/architecture/monolith.png)
 
 The following are different ways of running in either `Current` or `Microservice` architecture with either `Kusama` or `Polkadot`, and either `Development` or `Production`:
 
 - `Kusama Current`
   - Running as a monolith with production values
 - `Polkadot Current`
   - Running as a monolith with production values
-- `Kusama Microservice`
-  - Running as microservices with production values
-- `Polkadot Microservice`
-  - Running as microservices with production values
-- `Polkadot Current Dev`
-  - Running as a monolith with development values
-- `Kusama Current Dev`
-  - Running as a monolith with development values
-- `Kusama Microservice Dev`
-  - Running as microservices with development values
-- `Polkadot Microservice Dev`
-  - Running as microservices with development values
 
 Each package contains a `Dockerfile`, which is used for running in production,  and `Dockerfile-dev`, which is used for development. The development images will use run with `nodemon` so that each time files is saved/changed it will rebuild the image and restart the container. Any changes for the regular run `Dockerfile` will need a manual rebuilding of the docker image.
 
@@ -86,8 +64,7 @@ cd 1k-validators-be
 ### Installing System Dependencies
 
 Ensure the following are installed on your machine:
-- [Node.js](https://nodejs.org/en/) (v12 or higher)
-- [Yarn](https://yarnpkg.com/) (v1.22 or higher)
+- [Node.js](https://nodejs.org/en/) (v21 or higher)
 - [Docker](https://www.docker.com/) (v19 or higher)
 
 
@@ -121,30 +98,6 @@ Polkadot Current / Monolith Dev:
 yarn docker:polkadot-current-dev:start
 ```
 
-Kusama Microservice Production:
-
-```bash
-yarn docker:kusama-microscervice:start
-```
-
-Kusama Microservice Dev:
-
-```bash
-yarn docker:kusama-microservice-dev:start
-```
-
-Polkadot Microservice Production:
-
-```bash
-yarn docker:polkadot-current:start
-```
-
-Polkadot Microservice Dev:
-
-```bash
-yarn docker:polkadot-current-dev:start
-```
-
 ### Install Yarn Dependencies
 ```bash
 yarn install
@@ -170,50 +123,6 @@ This will create a configuration file for a Kusama instance that mirrors what is
 yarn create-config-polkadot-current
 ```
 
-Kusama Microservice Config:
-This will create configuration files for a Kusama instance for each microservice that runs with production values. This runs `core`, `gateway`, `telemetry`, and `worker` as separate processes in their own container - each one needs it's own configuration file.
-```bash
-yarn create-config-kusama-microservice
-```
-
-Polkadot Microservice Config:
-This will create configuration files for a Polkadot instance for each microservice that runs with production values. This runs `core`, `gateway`, `telemetry`, and `worker` as separate processes in their own container - each one needs it's own configuration file.
-```bash
-yarn create-config-polkadot-microservice
-```
-
-
-
-### Running the Microservices
-
-#### Running `Kusama Current` or `Polkadot Current`:
-
-Either is from the same `docker-compose.current.yml` file, and runs only the `core` container, `mongo` container, and `mongo-express` container.
-
-Build and run as detached daemon:
-```bash
-docker compose -f docker-compose.current.yml up -d --build
-```
-
-#### Running `Kusama Microservice` or `Polkadot Microservice`:
-
-Either is from the same `docker-compose.microservice.yml` file. This runs `core`, `gateway`, `telemetry`, and `worker` as separate processes in their own container - each one needs it's own configuration file. It additionally runs a `redis`, `mongo`, and `mongo-express` container.
-
-Build and run as detached daemon:
-```bash
-docker compose -f docker-compose.microservice.yml up -d --build
-```
-
-#### Running `Kusama Current Dev`, `Polkadot Current Dev`, `Kusama Microservice Dev`, or `Polkadot Microservice Dev`
-
-Either is from the same `docker-compose.yml` file.
-
-Build and run as detached daemon:
-```bash
-docker compose -f docker-compose.yml up -d --build
-```
-
-
 ### Viewing Logs
 
 To view the aggregated logs of all the containers:
@@ -297,9 +206,6 @@ You can then query an endpoint like `/candidates` by going to `http://localhost:
 
 To view the Mongo Express GUI to interact with the MongoDB Database, go to `http://localhost:8888/` in your browser. Or run `yarn open:mongo-express` from the root directory.
 
-#### BullMQ Board (Job Queue GUI)
-
-To view the BullMQ Board GUI to interact with the Job Queue, go to `http://localhost:3301/bull` in your browser if running as microservices. Or run `yarn open:bull` from the root directory.
 
 ## :pencil: Contribute
 

apps/1kv-backend-staging/templates/kusama-otv-backend.yaml

@@ -13,7 +13,7 @@ spec:
   # syncPolicy:
   #   automated:
   #     prune: true
-  #     selfHeal: true 
+  #     selfHeal: true
   source:
     repoURL: 'https://github.com/w3f/1k-validators-be.git'
     path: charts/otv-backend
@@ -46,27 +46,29 @@ spec:
                   "apiEndpoints": [
                     "wss://kusama-rpc-tn.dwellir.com",
                     "wss://kusama-rpc.dwellir.com",
-                    "wss://kusama.public.curie.radiumblock.xyz/ws",
+                    "wss://kusama.public.curie.radiumblock.co/ws",
                     "wss://rpc.ibp.network/kusama",
                     "wss://rpc.dotters.network/kusama",
                     "wss://ksm-rpc.stakeworld.io"
                   ],
+                  "apiPeopleEndpoints": ["wss://kusama-people-rpc.polkadot.io"],
                   "candidatesUrl": "https://raw.githubusercontent.com/w3f/1k-validators-be/master/candidates/kusama.json"
                 },
                 "constraints": {
                   "skipConnectionTime": true,
                   "skipIdentity": false,
-                  "skipClientUpgrade": false,
                   "skipUnclaimed": true,
+                  "clientUpgrade": {
+                    "skip": false
+                  },
                   "minSelfStake": 10000000000000,
                   "commission": 150000000,
                   "unclaimedEraThreshold": 4,
                   "sanctionedGeoArea": {
                     "skip": false,
                     "sanctionedCountries": ["RU", "IR", "CU", "KP", "SY"],
                     "sanctionedRegions": ["Crimea", "Autonomous Republic of Crimea", "Republic of Crimea", "Luhansk", "Luhanska Oblast", "Luhanska", "Luganskaya Oblast’", "Luganskaya", "Donetsk", "Donetska Oblast", "Donetskaya Oblast’", "Donetska", "Donetskaya", "Sevastopol City", "Sevastopol", "Gorod Sevastopol"]
-                  },
-                  "forceClientVersion": "v1.12.0"
+                  }
                 },
                 "cron": {
                   "monitor": "0 */15 * * * *",

apps/1kv-backend-staging/templates/polkadot-otv-backend.yaml

@@ -55,17 +55,18 @@ spec:
                 "constraints": {
                   "skipConnectionTime": false,
                   "skipIdentity": false,
-                  "skipClientUpgrade": false,
                   "skipUnclaimed": true,
+                  "clientUpgrade": {
+                    "skip": false
+                  },
                   "minSelfStake": 50000000000000,
                   "commission": 50000000,
                   "unclaimedEraThreshold": 1,
                   "sanctionedGeoArea": {
                     "skip": false,
                     "sanctionedCountries": ["RU", "IR", "CU", "KP", "SY"],
                     "sanctionedRegions": ["Crimea", "Autonomous Republic of Crimea", "Republic of Crimea", "Luhansk", "Luhanska Oblast", "Luhanska", "Luganskaya Oblast’", "Luganskaya", "Donetsk", "Donetska Oblast", "Donetskaya Oblast’", "Donetska", "Donetskaya", "Sevastopol City", "Sevastopol", "Gorod Sevastopol"]
-                  },
-                  "forceClientVersion": "v1.12.0"
+                  }
                 },
                 "cron": {
                   "monitor": "0 */15 * * * *",