diff --git a/cmd/bot/run.go b/cmd/bot/run.go index 6d7940d..0528a9d 100644 --- a/cmd/bot/run.go +++ b/cmd/bot/run.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math/big" + "net/http" "strings" "time" @@ -16,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/urfave/cli/v2" "gorm.io/driver/mysql" "gorm.io/gorm" @@ -61,10 +63,21 @@ func RunCommand(ctx *cli.Context) error { l2ScannedBlock, err := queryL2ScannedBlock(db, cfg.L2StartingNumber) if err != nil { - return err + return fmt.Errorf("failed to query l2_scanned_blocks: %w", err) + } else { + logger.Info("starting from block", "blockNumber", l2ScannedBlock.Number) } - logger.Info("starting from block", "blockNumber", l2ScannedBlock.Number) + go func() { + http.Handle("/metrics", promhttp.Handler()) + http.Handle("/debug/metrics/prometheus", promhttp.Handler()) + err := http.ListenAndServe(":6060", nil) + if err != nil { + logger.Error("failed to start prometheus server", "error", err) + } + }() + + go core.StartMetrics(ctx.Context, &cfg, &l1Client.Client, db, logger) go WatchBotDelegatedWithdrawals(ctx.Context, logger, db, l2Client, l2ScannedBlock, cfg) go ProcessBotDelegatedWithdrawals(ctx.Context, logger, db, l1Client, l2Client, cfg) diff --git a/core/metrics.go b/core/metrics.go new file mode 100644 index 0000000..56eac54 --- /dev/null +++ b/core/metrics.go @@ -0,0 +1,100 @@ +package core + +import ( + "context" + "errors" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "gorm.io/gorm" + "time" +) + +var ( + TxSignerBalance = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "opbnb_bridge_bot_tx_signer_balance", + Help: "The balance of the tx signer", + }) + + ScannedBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "opbnb_bridge_bot_scanned_block_number", + Help: "The block number that has been scanned", + }) + + UnprovenWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "opbnb_bridge_bot_unproven_withdrawals", + Help: "The number of unproven withdrawals", + }) + UnfinalizedWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "opbnb_bridge_bot_unfinalized_withdrawals", + Help: "The number of unfinalized withdrawals", + }) + EarliestUnProvenWithdrawalBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "opbnb_bridge_bot_earliest_unproven_withdrawal_block_number", + Help: "The earliest block number of unproven withdrawals", + }) + EarliestUnfinalizedWithdrawalBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "opbnb_bridge_bot_earliest_unfinalized_withdrawal_block_number", + Help: "The earliest block number of unfinalized withdrawals", + }) + + FailedWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "opbnb_bridge_bot_failed_withdrawals", + Help: "The number of failed withdrawals", + }) +) + +func StartMetrics(ctx context.Context, cfg *Config, l1Client *ethclient.Client, db *gorm.DB, logger log.Logger) { + ticker := time.NewTicker(5 * time.Second) + for range ticker.C { + _, signerAddress, _ := cfg.SignerKeyPair() + balance, err := l1Client.BalanceAt(ctx, *signerAddress, nil) + if err != nil { + logger.Error("failed to get signer balance", "error", err) + } + TxSignerBalance.Set(float64(balance.Int64())) + + var scannedBlock L2ScannedBlock + result := db.Last(&scannedBlock) + if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) { + logger.Error("failed to query scanned block", "error", result.Error) + } + ScannedBlockNumber.Set(float64(scannedBlock.Number)) + + var unprovenCnt int64 + result = db.Table("bot_delegated_withdrawals").Where("proven_time IS NULL AND failure_reason IS NULL").Count(&unprovenCnt) + if result.Error != nil { + logger.Error("failed to count withdrawals", "error", result.Error) + } + UnprovenWithdrawals.Set(float64(unprovenCnt)) + + var unfinalizedCnt int64 + result = db.Table("bot_delegated_withdrawals").Where("finalized_time IS NULL AND proven_time IS NOT NULL AND failure_reason IS NULL").Count(&unfinalizedCnt) + if result.Error != nil { + logger.Error("failed to count withdrawals", "error", result.Error) + } + UnfinalizedWithdrawals.Set(float64(unfinalizedCnt)) + + var failedCnt int64 + result = db.Table("bot_delegated_withdrawals").Where("failure_reason IS NOT NULL").Count(&failedCnt) + if result.Error != nil { + logger.Error("failed to count withdrawals", "error", result.Error) + } + FailedWithdrawals.Set(float64(failedCnt)) + + firstUnproven := BotDelegatedWithdrawal{} + result = db.Table("bot_delegated_withdrawals").Order("id asc").Where("proven_time IS NULL AND failure_reason IS NULL").First(&firstUnproven) + if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) { + logger.Error("failed to query withdrawals", "error", result.Error) + } + EarliestUnProvenWithdrawalBlockNumber.Set(float64(firstUnproven.InitiatedBlockNumber)) + + firstUnfinalized := BotDelegatedWithdrawal{} + result = db.Table("bot_delegated_withdrawals").Order("id asc").Where("finalized_time IS NULL AND proven_time IS NOT NULL AND failure_reason IS NULL").First(&firstUnfinalized) + if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) { + logger.Error("failed to query withdrawals", "error", result.Error) + } + EarliestUnfinalizedWithdrawalBlockNumber.Set(float64(firstUnfinalized.InitiatedBlockNumber)) + } +} diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c4db5b3..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.8' - -services: - mysql: - image: mysql:latest - environment: - - MYSQL_ROOT_PASSWORD=db_password - - MYSQL_DATABASE=db_name - - MYSQL_USER=db_username - - MYSQL_PASSWORD=db_password - command: --default-authentication-plugin=mysql_native_password - ports: - - "3306:3306" - volumes: - - mysql_data:/var/lib/mysql - -volumes: - mysql_data: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..e686168 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,43 @@ +version: '3.8' + +services: + mysql: + image: mysql:latest + environment: + - MYSQL_ROOT_PASSWORD=db_password + - MYSQL_DATABASE=db_name + - MYSQL_USER=db_username + - MYSQL_PASSWORD=db_password + command: --default-authentication-plugin=mysql_native_password + ports: + - "3306:3306" + volumes: + - mysql_data:/var/lib/mysql + + prometheus: + image: prom/prometheus + container_name: prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - 9090:9090 + restart: unless-stopped + volumes: + - ./prometheus:/etc/prometheus + - prom_data:/prometheus + + grafana: + image: grafana/grafana + container_name: grafana + ports: + - 3000:3000 + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=grafana + volumes: + - ./grafana:/etc/grafana/provisioning/datasources + +volumes: + mysql_data: + prom_data: diff --git a/docker/grafana/datasource.yml b/docker/grafana/datasource.yml new file mode 100644 index 0000000..d7b8286 --- /dev/null +++ b/docker/grafana/datasource.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy + editable: true diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml new file mode 100644 index 0000000..f48194d --- /dev/null +++ b/docker/prometheus/prometheus.yml @@ -0,0 +1,21 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - static_configs: + - targets: [] + scheme: http + timeout: 10s + api_version: v1 +scrape_configs: +- job_name: prometheus + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - localhost:9090