feat: new export for stuck packets

cmd/relayer_exporter/relayer_exporter.go

@@ -6,11 +6,12 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	"github.com/archway-network/relayer_exporter/pkg/collector"
 	"github.com/archway-network/relayer_exporter/pkg/config"
 	log "github.com/archway-network/relayer_exporter/pkg/logger"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
@@ -42,7 +43,14 @@ func main() {
 		log.Fatal(err.Error())
 	}
 
-	log.Info(fmt.Sprintf("Getting IBC paths from %s/%s/%s on GitHub", cfg.GitHub.Org, cfg.GitHub.Repo, cfg.GitHub.IBCDir))
+	log.Info(
+		fmt.Sprintf(
+			"Getting IBC paths from %s/%s/%s on GitHub",
+			cfg.GitHub.Org,
+			cfg.GitHub.Repo,
+			cfg.GitHub.IBCDir,
+		),
+	)
 
 	// TODO: Add a feature to refresh paths at configured interval
 	paths, err := cfg.IBCPaths()
@@ -52,7 +60,7 @@ func main() {
 
 	rpcs := cfg.GetRPCsMap()
 
-	clientsCollector := collector.IBCClientsCollector{
+	ibcCollector := collector.IBCCollector{
 		RPCs:  rpcs,
 		Paths: paths,
 	}
@@ -62,7 +70,7 @@ func main() {
 		Accounts: cfg.Accounts,
 	}
 
-	prometheus.MustRegister(clientsCollector)
+	prometheus.MustRegister(ibcCollector)
 	prometheus.MustRegister(balancesCollector)
 
 	http.Handle("/metrics", promhttp.Handler())

pkg/collector/collector.go

@@ -1,22 +1,27 @@
 package collector
 
 import (
+	"fmt"
 	"math/big"
 	"sync"
 
-	"github.com/archway-network/relayer_exporter/pkg/config"
-	"github.com/archway-network/relayer_exporter/pkg/ibc"
-	log "github.com/archway-network/relayer_exporter/pkg/logger"
 	"github.com/cosmos/relayer/v2/relayer"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
+
+	"github.com/archway-network/relayer_exporter/pkg/config"
+	"github.com/archway-network/relayer_exporter/pkg/ibc"
+	log "github.com/archway-network/relayer_exporter/pkg/logger"
 )
 
 const (
-	successStatus           = "success"
-	errorStatus             = "error"
-	clientExpiryMetricName  = "cosmos_ibc_client_expiry"
-	walletBalanceMetricName = "cosmos_wallet_balance"
+	successStatus                    = "success"
+	errorStatus                      = "error"
+	clientExpiryMetricName           = "cosmos_ibc_client_expiry"
+	walletBalanceMetricName          = "cosmos_wallet_balance"
+	channelStuckPacketsMetricName    = "cosmos_ibc_stuck_packets_total"
+	channelSrcStuckPacketsMetricName = "cosmos_ibc_stuck_packets_src"
+	channelDstStuckPacketsMetricName = "cosmos_ibc_stuck_packets_dst"
 )
 
 var (
@@ -25,14 +30,47 @@ var (
 		"Returns light client expiry in unixtime.",
 		[]string{"host_chain_id", "client_id", "target_chain_id", "status"}, nil,
 	)
+	channelStuckPackets = prometheus.NewDesc(
+		channelStuckPacketsMetricName,
+		"Returns stuck packets for a channel.",
+		[]string{
+			"channel_id",
+			"src_chain_id",
+			"dst_chain_id",
+			"status",
+		},
+		nil,
+	)
+	channelSrcStuckPackets = prometheus.NewDesc(
+		channelSrcStuckPacketsMetricName,
+		"Returns source stuck packets for a channel.",
+		[]string{
+			"channel_id",
+			"src_chain_id",
+			"dst_chain_id",
+			"status",
+		},
+		nil,
+	)
+	channelDstStuckPackets = prometheus.NewDesc(
+		channelDstStuckPacketsMetricName,
+		"Returns destination stuck packets for a channel.",
+		[]string{
+			"channel_id",
+			"src_chain_id",
+			"dst_chain_id",
+			"status",
+		},
+		nil,
+	)
 	walletBalance = prometheus.NewDesc(
 		walletBalanceMetricName,
 		"Returns wallet balance for an address on a chain.",
 		[]string{"account", "chain_id", "denom", "status"}, nil,
 	)
 )
 
-type IBCClientsCollector struct {
+type IBCCollector struct {
 	RPCs  *map[string]config.RPC
 	Paths []*relayer.IBCdata
 }
@@ -42,12 +80,19 @@ type WalletBalanceCollector struct {
 	Accounts []config.Account
 }
 
-func (cc IBCClientsCollector) Describe(ch chan<- *prometheus.Desc) {
+func (cc IBCCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- clientExpiry
+	ch <- channelStuckPackets
 }
 
-func (cc IBCClientsCollector) Collect(ch chan<- prometheus.Metric) {
-	log.Debug("Start collecting", zap.String("metric", clientExpiryMetricName))
+func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) {
+	log.Debug(
+		"Start collecting",
+		zap.String(
+			"metrics",
+			fmt.Sprintf("%s, %s", clientExpiryMetricName, channelStuckPacketsMetricName),
+		),
+	)
 
 	var wg sync.WaitGroup
 
@@ -66,6 +111,51 @@ func (cc IBCClientsCollector) Collect(ch chan<- prometheus.Metric) {
 				log.Error(err.Error())
 			}
 
+			stuckPackets, err := ibc.GetChannelInfo(path, cc.RPCs)
+			if err != nil {
+				status = errorStatus
+
+				log.Error(err.Error())
+			}
+
+			for _, sp := range stuckPackets.Channels {
+				ch <- prometheus.MustNewConstMetric(
+					channelStuckPackets,
+					prometheus.GaugeValue,
+					float64(sp.StuckPackets.Total),
+					[]string{
+						sp.Name,
+						(*cc.RPCs)[path.Chain1.ChainName].ChainID,
+						(*cc.RPCs)[path.Chain2.ChainName].ChainID,
+						status,
+					}...,
+				)
+
+				ch <- prometheus.MustNewConstMetric(
+					channelSrcStuckPackets,
+					prometheus.GaugeValue,
+					float64(sp.StuckPackets.Source),
+					[]string{
+						sp.Name,
+						(*cc.RPCs)[path.Chain1.ChainName].ChainID,
+						(*cc.RPCs)[path.Chain2.ChainName].ChainID,
+						status,
+					}...,
+				)
+
+				ch <- prometheus.MustNewConstMetric(
+					channelDstStuckPackets,
+					prometheus.GaugeValue,
+					float64(sp.StuckPackets.Destination),
+					[]string{
+						sp.Name,
+						(*cc.RPCs)[path.Chain1.ChainName].ChainID,
+						(*cc.RPCs)[path.Chain2.ChainName].ChainID,
+						status,
+					}...,
+				)
+			}
+
 			ch <- prometheus.MustNewConstMetric(
 				clientExpiry,
 				prometheus.GaugeValue,

pkg/ibc/ibc.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"time"
 
+	chantypes "github.com/cosmos/ibc-go/v7/modules/core/04-channel/types"
+	"github.com/cosmos/relayer/v2/relayer"
+
 	"github.com/archway-network/relayer_exporter/pkg/chain"
 	"github.com/archway-network/relayer_exporter/pkg/config"
-	"github.com/cosmos/relayer/v2/relayer"
 )
 
 type ClientsInfo struct {
@@ -19,6 +21,19 @@ type ClientsInfo struct {
 	ChainBClientExpiration time.Time
 }
 
+type ChannelsInfo struct {
+	Channels []Channel
+}
+
+type Channel struct {
+	Name         string
+	StuckPackets struct {
+		Source      int
+		Destination int
+		Total       int
+	}
+}
+
 func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsInfo, error) {
 	clientsInfo := ClientsInfo{}
 
@@ -50,15 +65,88 @@ func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsI
 
 	ctx := context.Background()
 
-	clientsInfo.ChainAClientExpiration, clientsInfo.ChainAClientInfo, err = relayer.QueryClientExpiration(ctx, chainA, chainB)
+	clientsInfo.ChainAClientExpiration, clientsInfo.ChainAClientInfo, err = relayer.QueryClientExpiration(
+		ctx,
+		chainA,
+		chainB,
+	)
 	if err != nil {
 		return ClientsInfo{}, fmt.Errorf("Error: %w path %v <-> %v", err, cdA, cdB)
 	}
 
-	clientsInfo.ChainBClientExpiration, clientsInfo.ChainBClientInfo, err = relayer.QueryClientExpiration(ctx, chainB, chainA)
+	clientsInfo.ChainBClientExpiration, clientsInfo.ChainBClientInfo, err = relayer.QueryClientExpiration(
+		ctx,
+		chainB,
+		chainA,
+	)
 	if err != nil {
 		return ClientsInfo{}, fmt.Errorf("Error: %w path %v <-> %v", err, cdB, cdA)
 	}
 
 	return clientsInfo, nil
 }
+
+func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelsInfo, error) {
+	ctx := context.Background()
+	channelInfo := ChannelsInfo{}
+
+	cdA := chain.Info{
+		ChainID:  (*rpcs)[ibc.Chain1.ChainName].ChainID,
+		RPCAddr:  (*rpcs)[ibc.Chain1.ChainName].URL,
+		ClientID: ibc.Chain1.ClientID,
+	}
+
+	chainA, err := chain.PrepChain(cdA)
+	if err != nil {
+		return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdA)
+	}
+
+	cdB := chain.Info{
+		ChainID:  (*rpcs)[ibc.Chain2.ChainName].ChainID,
+		RPCAddr:  (*rpcs)[ibc.Chain2.ChainName].URL,
+		ClientID: ibc.Chain2.ClientID,
+	}
+
+	chainB, err := chain.PrepChain(cdB)
+	if err != nil {
+		return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdB)
+	}
+
+	for _, c := range ibc.Channels {
+		var order chantypes.Order
+
+		var channel Channel
+
+		channel.Name = c.Chain1.ChannelID
+
+		switch c.Ordering {
+		case "none":
+			order = chantypes.NONE
+		case "unordered":
+			order = chantypes.UNORDERED
+		case "ordered":
+			order = chantypes.ORDERED
+		}
+
+		ch := chantypes.IdentifiedChannel{
+			State:    3,
+			Ordering: order,
+			Counterparty: chantypes.Counterparty{
+				PortId:    c.Chain2.PortID,
+				ChannelId: c.Chain2.ChannelID,
+			},
+			PortId:    c.Chain1.PortID,
+			ChannelId: c.Chain2.ChannelID,
+		}
+
+		unrelayedSequences := relayer.UnrelayedSequences(ctx, chainA, chainB, &ch)
+
+		channel.StuckPackets.Total += len(unrelayedSequences.Src) + len(unrelayedSequences.Dst)
+		channel.StuckPackets.Source += len(unrelayedSequences.Src)
+		channel.StuckPackets.Destination += len(unrelayedSequences.Dst)
+
+		channelInfo.Channels = append(channelInfo.Channels, channel)
+	}
+
+	return channelInfo, nil
+}