add metrics

cmd/bot/run.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"net/http"
 	"strings"
 	"time"
 
@@ -16,6 +17,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/urfave/cli/v2"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
@@ -61,10 +63,21 @@ func RunCommand(ctx *cli.Context) error {
 
 	l2ScannedBlock, err := queryL2ScannedBlock(db, cfg.L2StartingNumber)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to query l2_scanned_blocks: %w", err)
+	} else {
+		logger.Info("starting from block", "blockNumber", l2ScannedBlock.Number)
 	}
-	logger.Info("starting from block", "blockNumber", l2ScannedBlock.Number)
 
+	go func() {
+		http.Handle("/metrics", promhttp.Handler())
+		http.Handle("/debug/metrics/prometheus", promhttp.Handler())
+		err := http.ListenAndServe(":6060", nil)
+		if err != nil {
+			logger.Error("failed to start prometheus server", "error", err)
+		}
+	}()
+
+	go core.StartMetrics(ctx.Context, &cfg, &l1Client.Client, db, logger)
 	go WatchBotDelegatedWithdrawals(ctx.Context, logger, db, l2Client, l2ScannedBlock, cfg)
 	go ProcessBotDelegatedWithdrawals(ctx.Context, logger, db, l1Client, l2Client, cfg)
 

core/metrics.go

@@ -0,0 +1,100 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"gorm.io/gorm"
+	"time"
+)
+
+var (
+	TxSignerBalance = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "opbnb_bridge_bot_tx_signer_balance",
+		Help: "The balance of the tx signer",
+	})
+
+	ScannedBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "opbnb_bridge_bot_scanned_block_number",
+		Help: "The block number that has been scanned",
+	})
+
+	UnprovenWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "opbnb_bridge_bot_unproven_withdrawals",
+		Help: "The number of unproven withdrawals",
+	})
+	UnfinalizedWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "opbnb_bridge_bot_unfinalized_withdrawals",
+		Help: "The number of unfinalized withdrawals",
+	})
+	EarliestUnProvenWithdrawalBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "opbnb_bridge_bot_earliest_unproven_withdrawal_block_number",
+		Help: "The earliest block number of unproven withdrawals",
+	})
+	EarliestUnfinalizedWithdrawalBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "opbnb_bridge_bot_earliest_unfinalized_withdrawal_block_number",
+		Help: "The earliest block number of unfinalized withdrawals",
+	})
+
+	FailedWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "opbnb_bridge_bot_failed_withdrawals",
+		Help: "The number of failed withdrawals",
+	})
+)
+
+func StartMetrics(ctx context.Context, cfg *Config, l1Client *ethclient.Client, db *gorm.DB, logger log.Logger) {
+	ticker := time.NewTicker(5 * time.Second)
+	for range ticker.C {
+		_, signerAddress, _ := cfg.SignerKeyPair()
+		balance, err := l1Client.BalanceAt(ctx, *signerAddress, nil)
+		if err != nil {
+			logger.Error("failed to get signer balance", "error", err)
+		}
+		TxSignerBalance.Set(float64(balance.Int64()))
+
+		var scannedBlock L2ScannedBlock
+		result := db.Last(&scannedBlock)
+		if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			logger.Error("failed to query scanned block", "error", result.Error)
+		}
+		ScannedBlockNumber.Set(float64(scannedBlock.Number))
+
+		var unprovenCnt int64
+		result = db.Table("bot_delegated_withdrawals").Where("proven_time IS NULL AND failure_reason IS NULL").Count(&unprovenCnt)
+		if result.Error != nil {
+			logger.Error("failed to count withdrawals", "error", result.Error)
+		}
+		UnprovenWithdrawals.Set(float64(unprovenCnt))
+
+		var unfinalizedCnt int64
+		result = db.Table("bot_delegated_withdrawals").Where("finalized_time IS NULL AND proven_time IS NOT NULL AND failure_reason IS NULL").Count(&unfinalizedCnt)
+		if result.Error != nil {
+			logger.Error("failed to count withdrawals", "error", result.Error)
+		}
+		UnfinalizedWithdrawals.Set(float64(unfinalizedCnt))
+
+		var failedCnt int64
+		result = db.Table("bot_delegated_withdrawals").Where("failure_reason IS NOT NULL").Count(&failedCnt)
+		if result.Error != nil {
+			logger.Error("failed to count withdrawals", "error", result.Error)
+		}
+		FailedWithdrawals.Set(float64(failedCnt))
+
+		firstUnproven := BotDelegatedWithdrawal{}
+		result = db.Table("bot_delegated_withdrawals").Order("id asc").Where("proven_time IS NULL AND failure_reason IS NULL").First(&firstUnproven)
+		if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			logger.Error("failed to query withdrawals", "error", result.Error)
+		}
+		EarliestUnProvenWithdrawalBlockNumber.Set(float64(firstUnproven.InitiatedBlockNumber))
+
+		firstUnfinalized := BotDelegatedWithdrawal{}
+		result = db.Table("bot_delegated_withdrawals").Order("id asc").Where("finalized_time IS NULL AND proven_time IS NOT NULL AND failure_reason IS NULL").First(&firstUnfinalized)
+		if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			logger.Error("failed to query withdrawals", "error", result.Error)
+		}
+		EarliestUnfinalizedWithdrawalBlockNumber.Set(float64(firstUnfinalized.InitiatedBlockNumber))
+	}
+}

docker/docker-compose.yml

@@ -0,0 +1,43 @@
+version: '3.8'
+
+services:
+  mysql:
+    image: mysql:latest
+    environment:
+      - MYSQL_ROOT_PASSWORD=db_password
+      - MYSQL_DATABASE=db_name
+      - MYSQL_USER=db_username
+      - MYSQL_PASSWORD=db_password
+    command: --default-authentication-plugin=mysql_native_password
+    ports:
+      - "3306:3306"
+    volumes:
+      - mysql_data:/var/lib/mysql
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - 9090:9090
+    restart: unless-stopped
+    volumes:
+      - ./prometheus:/etc/prometheus
+      - prom_data:/prometheus
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    restart: unless-stopped
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=grafana
+    volumes:
+      - ./grafana:/etc/grafana/provisioning/datasources
+
+volumes:
+  mysql_data:
+  prom_data:

docker/grafana/datasource.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  url: http://prometheus:9090 
+  isDefault: true
+  access: proxy
+  editable: true

docker/prometheus/prometheus.yml

@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: []
+    scheme: http
+    timeout: 10s
+    api_version: v1
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - localhost:9090