Merge branch 'master' into fix/t_withdrawals-contains-only-one-withdr…

…awal-per-slot

.env-example

@@ -1,11 +1,20 @@
+# Clickhouse Server
+CH_NATIVE_PORT=9000
+CH_HTTP_PORT=8123
+CH_SQL_PORT=9005
+CH_USER=username
+CH_PASSWORD=password
+CH_DB=my_database
+CH_DB_HOST=localhost
+
 # Goteth Commons
-GOTETH_LOG_LEVEL="info" 
+GOTETH_LOG_LEVEL="info"
 GOTETH_BN_ENDPOINT="http://localhost:5052"
-GOTETH_DB_URL="clickhouse://db_user:db_password@db_ip:db_port/db_name?x-multi-statement=true"
+GOTETH_DB_URL="clickhouse://$CH_USER:$CH_PASSWORD@$CH_DB_HOST:$CH_NATIVE_PORT/$CH_DB?x-multi-statement=true"
 
 # Analyzer
 GOTETH_ANALYZER_EL_ENDPOINT="http://localhost:8545"
-GOTETH_ANALYZER_INIT_SLOT="10000" 
+GOTETH_ANALYZER_INIT_SLOT="10000"
 GOTETH_ANALYZER_FINAL_SLOT="10050"
 GOTETH_ANALYZER_WORKERS_NUM="4"
 GOTETH_ANALYZER_DB_WORKERS_NUM="4"
@@ -15,10 +24,3 @@ GOTETH_ANALYZER_PROMETHEUS_PORT=9081
 
 # Validator Window
 GOTETH_VAL_WINDOW_NUM_EPOCHS=1
-
-# Clickhouse Server
-CH_NATIVE_PORT=9000
-CH_HTTP_PORT=8123
-CH_USER=username
-CH_PASSWORD=password
-CH_DB=my_database

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "go-relay-client"]
 	path = go-relay-client
-	url = [email protected]:migalabs/go-relay-client.git
+	url = https://github.com/migalabs/go-relay-client.git
+

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM golang:1.20-alpine as builder
+FROM golang:1.21-alpine as builder
 RUN apk add --update git gcc g++ openssh-client make
 WORKDIR /app
 COPY go.mod go.sum ./

README.md

@@ -4,27 +4,33 @@ GotEth is a go-written client that indexes all validator-related duties and para
 
 The client indexes all the validator/epoch related metrics into a set of postgreSQL tables which later on can be used to monitor the performance of validators in the beaconchain.
 
-This tool has been used to power the 
+This tool has been used to power the
+
 - [pandametrics.xyz](https://pandametrics.xyz/) public dashboard
 - [ethseer.io](https://ethseer.io) public dashboard
 
 ## Prerequisites
+
 To use the tool, the following requirements need to be installed in the machine:
-- [go](https://go.dev/doc/install) preferably on its 1.20 version or above. Go also needs to be executable from the terminal.
+
+- [go](https://go.dev/doc/install) preferably on its 1.21 version or above. Go also needs to be executable from the terminal.
 - Clickhouse DB
 - Access to an Ethereum CL beacon node (preferably an archive node to index the slots faster)
 - Access to an Ethereum execution node (optional)
 - Access to a Clickhouse server database (use native port, usually 9000)
 
 ## Installation
+
 The repository provides a Makefile that will take care of all your problems.
 
 To compile locally the client, just type the following command at the root of the directory:
+
 ```
 make build
 ```
 
 Or if you prefer to install the client locally type:
+
 ```
 make install
 ```
@@ -40,25 +46,29 @@ make install
 ## Download mode
 
 - Historical: this mode loops over slots between `initSlot` and `finalSlot`, which are configurable. Once all slots have been analyzed, the tool finishes the execution.
-- Finalized: `initSlot` and `finalSlot` are ignored. The tool starts the historical mode from the database last slot to the current head (beacon node) and then follows the chain head. To do this, the tool subscribes to `head` events. See [here](https://ethereum.github.io/beacon-APIs/#/Events/eventstream) for more information. 
+- Finalized: `initSlot` and `finalSlot` are ignored. The tool starts the historical mode from the database last slot to the current head (beacon node) and then follows the chain head. To do this, the tool subscribes to `head` events. See [here](https://ethereum.github.io/beacon-APIs/#/Events/eventstream) for more information.
 
 ## Running the tool
+
 To execute the tool, you can simply modify the `.env` file with your own configuration.
 
-*Running the tool (configurable in the `.env` file)*:
+_Running the tool (configurable in the `.env` file)_:
+
 ```
 docker-compose up goteth
 ```
 
-*Available Commands*:
+_Available Commands_:
+
 ```
 COMMANDS:
    blocks   analyze the Beacon Block of a given slot range
    val-window Removes old rows from the validator rewards table according to given parameters
    help, h  Shows a list of commands or help for one command
 ```
 
-*Available Options (configurable in the `.env` file)*
+_Available Options (configurable in the `.env` file)_
+
 ```
 
 Blocks
@@ -72,18 +82,19 @@ OPTIONS:
    --workers-num value     example: 3 (default: 4)
    --db-workers-num value  example: 3 (default: 4)
    --download-mode value   example: hybrid,historical,finalized. Default: hybrid
-   --metrics value         example: epoch,block,rewards,transactions. Empty for all (default: epoch,block)
+   --metrics value         example: epoch,block,rewards,transactions,api_rewards. Empty for all (default: epoch,block)
    --prometheus-port value Port on which to expose prometheus metrics (default: 9081)
    --help, -h              show help (default: false)
 ```
 
 ### Validator window (experimental)
 
-Validator rewards represent 95% of the disk usage of the database. When activated, the database grows very big, sometimes becoming too much data. 
+Validator rewards represent 95% of the disk usage of the database. When activated, the database grows very big, sometimes becoming too much data.
 We have developed a subcommand of the tool which maintains the last n epochs of rewards data in the database, prunning from the defined threshold backwards. So, one can configure the tool to maintain the last 100 epochs of data in the database, while prunning the rest.
 The pruning only affects the `t_validator_rewards_summary` table.
 
 Simply configure `GOTETH_VAL_WINDOW_NUM_EPOCHS` variable and run
+
 ```
 docker-compose up val-window
 ```
@@ -96,17 +107,19 @@ Keep in mind `api_rewards` data also downloads block rewards from the Beacon API
 
 In case you encounter any issue with the database, you can force the database version using the golang-migrate command line. Please refer [here](https://github.com/golang-migrate/migrate) for more information.
 More specifically, one could clean the migrations by forcing the version with <br>
-```migrate -path / -database "postgresql://username:secretkey@localhost:5432/database_name?sslmode=disable" force <current_version>``` <br>
+`migrate -path / -database "postgresql://username:secretkey@localhost:5432/database_name?sslmode=disable" force <current_version>` <br>
 If specific upgrades or downgrades need to be done manually, one could do this with <br>
-```migrate -path database/migration/ -database "postgresql://username:secretkey@localhost:5432/database_name?sslmode=disable" -verbose up```
-
+`migrate -path database/migration/ -database "postgresql://username:secretkey@localhost:5432/database_name?sslmode=disable" -verbose up`
 
 # From PostgreSQL to Clickhouse
+
 During `v3.0.0` we will migrate our database system from PostgreSQL to Clickhouse.
 If you wish to migrate your existing database, please follow [this](https://migalabs.notion.site/PostgreSQL-to-Clickhouse-migration-611a52a457824cd494d701773365f62f) guide.
 
 # Maintainers
+
 @cortze @tdahar
 
 # Contributing
-The project is open for everyone to contribute! 
+
+The project is open for everyone to contribute!

cmd/blocks_cmd.go

@@ -76,7 +76,7 @@ var BlocksCommand = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:        "metrics",
-			Usage:       "Metrics to be persisted to the database: epoch,block,rewards,transactions",
+			Usage:       "Metrics to be persisted to the database: epoch,block,rewards,transactions,api_rewards",
 			EnvVars:     []string{"ANALYZER_METRICS"},
 			DefaultText: "epoch,block",
 		},

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.7'
+version: "3.7"
 
 services:
   goteth:
@@ -19,11 +19,12 @@ services:
       --download-mode=${GOTETH_ANALYZER_DOWNLOAD_MODE}
       --metrics=${GOTETH_ANALYZER_METRICS}
       --prometheus-port=${GOTETH_ANALYZER_PROMETHEUS_PORT:-9081}
-    network_mode: 'host'
+    network_mode: "host"
+    restart: "always"
     depends_on:
       clickhouse:
         condition: service_healthy
-    
+
   val-window:
     build:
       context: ./
@@ -35,8 +36,9 @@ services:
       --bn-endpoint=${GOTETH_BN_ENDPOINT}
       --db-url=${GOTETH_DB_URL}
       --num-epochs=${GOTETH_VAL_WINDOW_NUM_EPOCHS:-1}
-    network_mode: 'host'
-
+    network_mode: "host"
+    restart: "always"
+
   clickhouse:
     image: clickhouse/clickhouse-server:24
     user: "$UID:$GID"
@@ -55,4 +57,4 @@ services:
       - "127.0.0.1:${CH_SQL_PORT:-9005}:9005"
     healthcheck: # executed inside the container
       test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping
-      interval: 5s
+      interval: 5s

go.mod

@@ -1,11 +1,12 @@
 module github.com/migalabs/goteth
 
-go 1.19
+go 1.21
 
 require (
 	github.com/ClickHouse/ch-go v0.61.0
 	github.com/ClickHouse/clickhouse-go/v2 v2.16.0
 	github.com/attestantio/go-eth2-client v0.19.10
+	github.com/attestantio/go-relay-client v0.2.5
 	github.com/golang-migrate/migrate/v4 v4.16.2
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb
 	github.com/pkg/errors v0.9.1
@@ -20,7 +21,6 @@ require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/attestantio/go-builder-client v0.2.3 // indirect
-	github.com/attestantio/go-relay-client v0.2.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
@@ -102,4 +102,4 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
-replace github.com/attestantio/go-relay-client => ./go-relay-client
+replace github.com/attestantio/go-relay-client => ./go-relay-client