From 1fec18540b5b6d7913425ed2d82ab4bbbee04e6c Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Thu, 26 Oct 2023 13:06:03 +0300 Subject: [PATCH 01/13] feat: combine all IBC messages to one collector --- cmd/relayer_exporter/relayer_exporter.go | 18 +++++-- pkg/collector/collector.go | 66 ++++++++++++++++++++---- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/cmd/relayer_exporter/relayer_exporter.go b/cmd/relayer_exporter/relayer_exporter.go index b9b19a7..4176d73 100644 --- a/cmd/relayer_exporter/relayer_exporter.go +++ b/cmd/relayer_exporter/relayer_exporter.go @@ -6,11 +6,12 @@ import ( "net/http" "os" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/archway-network/relayer_exporter/pkg/collector" "github.com/archway-network/relayer_exporter/pkg/config" log "github.com/archway-network/relayer_exporter/pkg/logger" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -42,7 +43,14 @@ func main() { log.Fatal(err.Error()) } - log.Info(fmt.Sprintf("Getting IBC paths from %s/%s/%s on GitHub", cfg.GitHub.Org, cfg.GitHub.Repo, cfg.GitHub.IBCDir)) + log.Info( + fmt.Sprintf( + "Getting IBC paths from %s/%s/%s on GitHub", + cfg.GitHub.Org, + cfg.GitHub.Repo, + cfg.GitHub.IBCDir, + ), + ) // TODO: Add a feature to refresh paths at configured interval paths, err := cfg.IBCPaths() @@ -52,7 +60,7 @@ func main() { rpcs := cfg.GetRPCsMap() - clientsCollector := collector.IBCClientsCollector{ + ibcCollector := collector.IBCCollector{ RPCs: rpcs, Paths: paths, } @@ -62,7 +70,7 @@ func main() { Accounts: cfg.Accounts, } - prometheus.MustRegister(clientsCollector) + prometheus.MustRegister(ibcCollector) prometheus.MustRegister(balancesCollector) http.Handle("/metrics", promhttp.Handler()) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 3c1f6a6..63b7f4a 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -1,22 +1,25 @@ package collector import ( + "fmt" "math/big" "sync" - "github.com/archway-network/relayer_exporter/pkg/config" - "github.com/archway-network/relayer_exporter/pkg/ibc" - log "github.com/archway-network/relayer_exporter/pkg/logger" "github.com/cosmos/relayer/v2/relayer" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" + + "github.com/archway-network/relayer_exporter/pkg/config" + "github.com/archway-network/relayer_exporter/pkg/ibc" + log "github.com/archway-network/relayer_exporter/pkg/logger" ) const ( - successStatus = "success" - errorStatus = "error" - clientExpiryMetricName = "cosmos_ibc_client_expiry" - walletBalanceMetricName = "cosmos_wallet_balance" + successStatus = "success" + errorStatus = "error" + clientExpiryMetricName = "cosmos_ibc_client_expiry" + walletBalanceMetricName = "cosmos_wallet_balance" + channelStuckPacketsMetricName = "cosmos_ibc_stuck_packets" ) var ( @@ -25,6 +28,19 @@ var ( "Returns light client expiry in unixtime.", []string{"host_chain_id", "client_id", "target_chain_id", "status"}, nil, ) + channelStuckPackets = prometheus.NewDesc( + channelStuckPacketsMetricName, + "Returns stuck packets for a channel.", + []string{ + "src_chain_id", + "channel_id", + "target_chain_id", + "source_stuck_packets", + "destination_stuck_packets", + "status", + }, + nil, + ) walletBalance = prometheus.NewDesc( walletBalanceMetricName, "Returns wallet balance for an address on a chain.", @@ -32,7 +48,7 @@ var ( ) ) -type IBCClientsCollector struct { +type IBCCollector struct { RPCs *map[string]config.RPC Paths []*relayer.IBCdata } @@ -42,12 +58,19 @@ type WalletBalanceCollector struct { Accounts []config.Account } -func (cc IBCClientsCollector) Describe(ch chan<- *prometheus.Desc) { +func (cc IBCCollector) Describe(ch chan<- *prometheus.Desc) { ch <- clientExpiry + ch <- channelStuckPackets } -func (cc IBCClientsCollector) Collect(ch chan<- prometheus.Metric) { - log.Debug("Start collecting", zap.String("metric", clientExpiryMetricName)) +func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { + log.Debug( + "Start collecting", + zap.String( + "metrics", + fmt.Sprintf("%s, %s", clientExpiryMetricName, channelStuckPacketsMetricName), + ), + ) var wg sync.WaitGroup @@ -66,6 +89,27 @@ func (cc IBCClientsCollector) Collect(ch chan<- prometheus.Metric) { log.Error(err.Error()) } + sp, err := ibc.GetChannelInfo(path, cc.RPCs) + if err != nil { + status = errorStatus + + log.Error(err.Error()) + } + + ch <- prometheus.MustNewConstMetric( + channelStuckPackets, + prometheus.GaugeValue, + float64(sp.StuckPackets.Source+sp.StuckPackets.Destination), + []string{ + (*cc.RPCs)[path.Chain1.ChainName].ChainID, + path.Channels[0].Chain1.ChannelID, + (*cc.RPCs)[path.Chain2.ChainName].ChainID, + fmt.Sprintf("%d", sp.StuckPackets.Source), + fmt.Sprintf("%d", sp.StuckPackets.Destination), + status, + }..., + ) + ch <- prometheus.MustNewConstMetric( clientExpiry, prometheus.GaugeValue, From 54ffe5d35dfd3412cf7073b650b77fe3b6f65117 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Thu, 26 Oct 2023 13:06:59 +0300 Subject: [PATCH 02/13] feat: add new IBC collector - ibc channel This is used to have metrics about IBC channel, now just have stuck packets --- pkg/ibc/ibc.go | 80 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index 196b8ff..772530a 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -5,9 +5,11 @@ import ( "fmt" "time" + chantypes "github.com/cosmos/ibc-go/v7/modules/core/04-channel/types" + "github.com/cosmos/relayer/v2/relayer" + "github.com/archway-network/relayer_exporter/pkg/chain" "github.com/archway-network/relayer_exporter/pkg/config" - "github.com/cosmos/relayer/v2/relayer" ) type ClientsInfo struct { @@ -19,6 +21,13 @@ type ClientsInfo struct { ChainBClientExpiration time.Time } +type ChannelInfo struct { + StuckPackets struct { + Source int + Destination int + } +} + func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsInfo, error) { clientsInfo := ClientsInfo{} @@ -50,15 +59,80 @@ func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsI ctx := context.Background() - clientsInfo.ChainAClientExpiration, clientsInfo.ChainAClientInfo, err = relayer.QueryClientExpiration(ctx, chainA, chainB) + clientsInfo.ChainAClientExpiration, clientsInfo.ChainAClientInfo, err = relayer.QueryClientExpiration( + ctx, + chainA, + chainB, + ) if err != nil { return ClientsInfo{}, fmt.Errorf("Error: %w path %v <-> %v", err, cdA, cdB) } - clientsInfo.ChainBClientExpiration, clientsInfo.ChainBClientInfo, err = relayer.QueryClientExpiration(ctx, chainB, chainA) + clientsInfo.ChainBClientExpiration, clientsInfo.ChainBClientInfo, err = relayer.QueryClientExpiration( + ctx, + chainB, + chainA, + ) if err != nil { return ClientsInfo{}, fmt.Errorf("Error: %w path %v <-> %v", err, cdB, cdA) } return clientsInfo, nil } + +func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelInfo, error) { + ctx := context.Background() + channelInfo := ChannelInfo{} + + cdA := chain.Info{ + ChainID: (*rpcs)[ibc.Chain1.ChainName].ChainID, + RPCAddr: (*rpcs)[ibc.Chain1.ChainName].URL, + ClientID: ibc.Chain1.ClientID, + } + + chainA, err := chain.PrepChain(cdA) + if err != nil { + return ChannelInfo{}, fmt.Errorf("Error: %w for %v", err, cdA) + } + + cdB := chain.Info{ + ChainID: (*rpcs)[ibc.Chain2.ChainName].ChainID, + RPCAddr: (*rpcs)[ibc.Chain2.ChainName].URL, + ClientID: ibc.Chain2.ClientID, + } + + chainB, err := chain.PrepChain(cdB) + if err != nil { + return ChannelInfo{}, fmt.Errorf("Error: %w for %v", err, cdB) + } + + for _, c := range ibc.Channels { + var order chantypes.Order + + switch c.Ordering { + case "none": + order = chantypes.NONE + case "unordered": + order = chantypes.UNORDERED + case "ordered": + order = chantypes.ORDERED + } + + channel := chantypes.IdentifiedChannel{ + State: 3, + Ordering: order, + Counterparty: chantypes.Counterparty{ + PortId: c.Chain2.PortID, + ChannelId: c.Chain2.ChannelID, + }, + PortId: c.Chain1.PortID, + ChannelId: c.Chain2.ChannelID, + } + + unrelayedSequences := relayer.UnrelayedSequences(ctx, chainA, chainB, &channel) + channelInfo.StuckPackets.Source += len(unrelayedSequences.Src) + channelInfo.StuckPackets.Destination += len(unrelayedSequences.Dst) + } + + return channelInfo, nil +} From fac6c90b9b8582f5659bf58f2bef5437596197eb Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Fri, 27 Oct 2023 10:56:17 +0300 Subject: [PATCH 03/13] feat: split one metric to three this splits one total metric so that it will produce three metrics, src,dst and total --- pkg/collector/collector.go | 90 ++++++++++++++++++++++++++++---------- pkg/ibc/ibc.go | 32 ++++++++++---- 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 63b7f4a..6caab59 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -15,11 +15,13 @@ import ( ) const ( - successStatus = "success" - errorStatus = "error" - clientExpiryMetricName = "cosmos_ibc_client_expiry" - walletBalanceMetricName = "cosmos_wallet_balance" - channelStuckPacketsMetricName = "cosmos_ibc_stuck_packets" + successStatus = "success" + errorStatus = "error" + clientExpiryMetricName = "cosmos_ibc_client_expiry" + walletBalanceMetricName = "cosmos_wallet_balance" + channelStuckPacketsMetricName = "cosmos_ibc_stuck_packets_total" + channelSrcStuckPacketsMetricName = "cosmos_ibc_stuck_packets_src" + channelDstStuckPacketsMetricName = "cosmos_ibc_stuck_packets_dst" ) var ( @@ -32,11 +34,31 @@ var ( channelStuckPacketsMetricName, "Returns stuck packets for a channel.", []string{ + "channel_id", "src_chain_id", + "dst_chain_id", + "status", + }, + nil, + ) + channelSrcStuckPackets = prometheus.NewDesc( + channelSrcStuckPacketsMetricName, + "Returns source stuck packets for a channel.", + []string{ + "channel_id", + "src_chain_id", + "dst_chain_id", + "status", + }, + nil, + ) + channelDstStuckPackets = prometheus.NewDesc( + channelDstStuckPacketsMetricName, + "Returns destination stuck packets for a channel.", + []string{ "channel_id", - "target_chain_id", - "source_stuck_packets", - "destination_stuck_packets", + "src_chain_id", + "dst_chain_id", "status", }, nil, @@ -89,26 +111,50 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { log.Error(err.Error()) } - sp, err := ibc.GetChannelInfo(path, cc.RPCs) + stuckPackets, err := ibc.GetChannelInfo(path, cc.RPCs) if err != nil { status = errorStatus log.Error(err.Error()) } - ch <- prometheus.MustNewConstMetric( - channelStuckPackets, - prometheus.GaugeValue, - float64(sp.StuckPackets.Source+sp.StuckPackets.Destination), - []string{ - (*cc.RPCs)[path.Chain1.ChainName].ChainID, - path.Channels[0].Chain1.ChannelID, - (*cc.RPCs)[path.Chain2.ChainName].ChainID, - fmt.Sprintf("%d", sp.StuckPackets.Source), - fmt.Sprintf("%d", sp.StuckPackets.Destination), - status, - }..., - ) + for _, sp := range stuckPackets.Channels { + ch <- prometheus.MustNewConstMetric( + channelStuckPackets, + prometheus.GaugeValue, + float64(sp.StuckPackets.Total), + []string{ + sp.Name, + (*cc.RPCs)[path.Chain1.ChainName].ChainID, + (*cc.RPCs)[path.Chain2.ChainName].ChainID, + status, + }..., + ) + + ch <- prometheus.MustNewConstMetric( + channelSrcStuckPackets, + prometheus.GaugeValue, + float64(sp.StuckPackets.Source), + []string{ + sp.Name, + (*cc.RPCs)[path.Chain1.ChainName].ChainID, + (*cc.RPCs)[path.Chain2.ChainName].ChainID, + status, + }..., + ) + + ch <- prometheus.MustNewConstMetric( + channelDstStuckPackets, + prometheus.GaugeValue, + float64(sp.StuckPackets.Destination), + []string{ + sp.Name, + (*cc.RPCs)[path.Chain1.ChainName].ChainID, + (*cc.RPCs)[path.Chain2.ChainName].ChainID, + status, + }..., + ) + } ch <- prometheus.MustNewConstMetric( clientExpiry, diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index 772530a..ab0b83b 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -21,10 +21,16 @@ type ClientsInfo struct { ChainBClientExpiration time.Time } -type ChannelInfo struct { +type ChannelsInfo struct { + Channels []Channel +} + +type Channel struct { + Name string StuckPackets struct { Source int Destination int + Total int } } @@ -80,9 +86,9 @@ func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsI return clientsInfo, nil } -func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelInfo, error) { +func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelsInfo, error) { ctx := context.Background() - channelInfo := ChannelInfo{} + channelInfo := ChannelsInfo{} cdA := chain.Info{ ChainID: (*rpcs)[ibc.Chain1.ChainName].ChainID, @@ -92,7 +98,7 @@ func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelI chainA, err := chain.PrepChain(cdA) if err != nil { - return ChannelInfo{}, fmt.Errorf("Error: %w for %v", err, cdA) + return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdA) } cdB := chain.Info{ @@ -103,12 +109,16 @@ func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelI chainB, err := chain.PrepChain(cdB) if err != nil { - return ChannelInfo{}, fmt.Errorf("Error: %w for %v", err, cdB) + return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdB) } for _, c := range ibc.Channels { var order chantypes.Order + var channel Channel + + channel.Name = c.Chain1.ChannelID + switch c.Ordering { case "none": order = chantypes.NONE @@ -118,7 +128,7 @@ func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelI order = chantypes.ORDERED } - channel := chantypes.IdentifiedChannel{ + ch := chantypes.IdentifiedChannel{ State: 3, Ordering: order, Counterparty: chantypes.Counterparty{ @@ -129,9 +139,13 @@ func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelI ChannelId: c.Chain2.ChannelID, } - unrelayedSequences := relayer.UnrelayedSequences(ctx, chainA, chainB, &channel) - channelInfo.StuckPackets.Source += len(unrelayedSequences.Src) - channelInfo.StuckPackets.Destination += len(unrelayedSequences.Dst) + unrelayedSequences := relayer.UnrelayedSequences(ctx, chainA, chainB, &ch) + + channel.StuckPackets.Total += len(unrelayedSequences.Src) + len(unrelayedSequences.Dst) + channel.StuckPackets.Source += len(unrelayedSequences.Src) + channel.StuckPackets.Destination += len(unrelayedSequences.Dst) + + channelInfo.Channels = append(channelInfo.Channels, channel) } return channelInfo, nil From b6da3ef83806969f020b87d3f59cc0a9aaaba214 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Fri, 27 Oct 2023 20:49:25 +0300 Subject: [PATCH 04/13] fix: error if config does not have IBC RPC information --- config.yaml | 2 +- pkg/collector/collector.go | 28 ++++++++++++++-------------- pkg/ibc/ibc.go | 6 ++++++ 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/config.yaml b/config.yaml index 596b14d..5eb3574 100644 --- a/config.yaml +++ b/config.yaml @@ -35,7 +35,7 @@ rpc: url: https://noble-rpc.polkachu.com:443 - chainName: nois chainId: nois-1 - url: https://rpc.cosmos.directory/nois:443 + url: https://nois.rpc.kjnodes.com:443 - chainName: osmosistestnet chainId: osmo-test-5 url: https://rpc.osmotest5.osmosis.zone:443 diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 6caab59..f0b73a9 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -111,6 +111,20 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { log.Error(err.Error()) } + ch <- prometheus.MustNewConstMetric( + clientExpiry, + prometheus.GaugeValue, + float64(ci.ChainAClientExpiration.Unix()), + []string{(*cc.RPCs)[path.Chain1.ChainName].ChainID, path.Chain1.ClientID, (*cc.RPCs)[path.Chain2.ChainName].ChainID, status}..., + ) + + ch <- prometheus.MustNewConstMetric( + clientExpiry, + prometheus.GaugeValue, + float64(ci.ChainBClientExpiration.Unix()), + []string{(*cc.RPCs)[path.Chain2.ChainName].ChainID, path.Chain2.ClientID, (*cc.RPCs)[path.Chain1.ChainName].ChainID, status}..., + ) + stuckPackets, err := ibc.GetChannelInfo(path, cc.RPCs) if err != nil { status = errorStatus @@ -155,20 +169,6 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { }..., ) } - - ch <- prometheus.MustNewConstMetric( - clientExpiry, - prometheus.GaugeValue, - float64(ci.ChainAClientExpiration.Unix()), - []string{(*cc.RPCs)[path.Chain1.ChainName].ChainID, path.Chain1.ClientID, (*cc.RPCs)[path.Chain2.ChainName].ChainID, status}..., - ) - - ch <- prometheus.MustNewConstMetric( - clientExpiry, - prometheus.GaugeValue, - float64(ci.ChainBClientExpiration.Unix()), - []string{(*cc.RPCs)[path.Chain2.ChainName].ChainID, path.Chain2.ClientID, (*cc.RPCs)[path.Chain1.ChainName].ChainID, status}..., - ) }(p) } diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index ab0b83b..da5dd64 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -90,6 +90,12 @@ func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channels ctx := context.Background() channelInfo := ChannelsInfo{} + if (*rpcs)[ibc.Chain1.ChainName].ChainID == "" || (*rpcs)[ibc.Chain2.ChainName].ChainID == "" { + return ChannelsInfo{}, fmt.Errorf( + "Error: RPC data is missing, cannot retrieve channel data", + ) + } + cdA := chain.Info{ ChainID: (*rpcs)[ibc.Chain1.ChainName].ChainID, RPCAddr: (*rpcs)[ibc.Chain1.ChainName].URL, From 248f8b9a94414a490e25fbbb208c13c4f836162e Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Mon, 30 Oct 2023 15:11:34 +0200 Subject: [PATCH 05/13] fix: remove magic number and have stateOpen const --- pkg/ibc/ibc.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index da5dd64..bf87060 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -12,6 +12,8 @@ import ( "github.com/archway-network/relayer_exporter/pkg/config" ) +const stateOpen = 3 + type ClientsInfo struct { ChainA *relayer.Chain ChainAClientInfo relayer.ClientStateInfo @@ -135,7 +137,7 @@ func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channels } ch := chantypes.IdentifiedChannel{ - State: 3, + State: stateOpen, Ordering: order, Counterparty: chantypes.Counterparty{ PortId: c.Chain2.PortID, From 839bdaa8850d5da19d37b870c4e7a08611a514e8 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Mon, 30 Oct 2023 15:58:21 +0200 Subject: [PATCH 06/13] chore: change naming scheme --- pkg/collector/collector.go | 2 +- pkg/ibc/ibc.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index f0b73a9..af54356 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -125,7 +125,7 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { []string{(*cc.RPCs)[path.Chain2.ChainName].ChainID, path.Chain2.ClientID, (*cc.RPCs)[path.Chain1.ChainName].ChainID, status}..., ) - stuckPackets, err := ibc.GetChannelInfo(path, cc.RPCs) + stuckPackets, err := ibc.GetChannelsInfo(path, cc.RPCs) if err != nil { status = errorStatus diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index bf87060..2f16ed4 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -88,7 +88,7 @@ func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsI return clientsInfo, nil } -func GetChannelInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelsInfo, error) { +func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelsInfo, error) { ctx := context.Background() channelInfo := ChannelsInfo{} From 8229fed8665306b15ce43fe2b2dfd762ef3e68b3 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Mon, 30 Oct 2023 16:52:49 +0200 Subject: [PATCH 07/13] chore: have path info in the error message --- pkg/ibc/ibc.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index 2f16ed4..42b5a0a 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -94,7 +94,8 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel if (*rpcs)[ibc.Chain1.ChainName].ChainID == "" || (*rpcs)[ibc.Chain2.ChainName].ChainID == "" { return ChannelsInfo{}, fmt.Errorf( - "Error: RPC data is missing, cannot retrieve channel data", + "Error: RPC data is missing, cannot retrieve channel data: %v", + ibc.Channels, ) } From 8ce481c866d7743f86bc1fc4772284f907d2d869 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Mon, 30 Oct 2023 17:17:44 +0200 Subject: [PATCH 08/13] fix: ensure that all RPC endpoints are working --- pkg/ibc/ibc.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index 42b5a0a..a9ea576 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -121,6 +121,13 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdB) } + // test that RPC endpoints are working + if _, _, err := relayer.QueryLatestHeights( + ctx, chainA, chainB, + ); err != nil { + return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdA) + } + for _, c := range ibc.Channels { var order chantypes.Order From bb9794ccb6f4ce4e2aa424f6f563aa9c4167c871 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Mon, 30 Oct 2023 17:45:54 +0200 Subject: [PATCH 09/13] feat: have src, dst channel IDs --- pkg/collector/collector.go | 19 +++++++++++++------ pkg/ibc/ibc.go | 6 ++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index af54356..7e5c1a4 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -34,7 +34,8 @@ var ( channelStuckPacketsMetricName, "Returns stuck packets for a channel.", []string{ - "channel_id", + "src_channel_id", + "dst_channel_id", "src_chain_id", "dst_chain_id", "status", @@ -45,7 +46,8 @@ var ( channelSrcStuckPacketsMetricName, "Returns source stuck packets for a channel.", []string{ - "channel_id", + "src_channel_id", + "dst_channel_id", "src_chain_id", "dst_chain_id", "status", @@ -56,7 +58,8 @@ var ( channelDstStuckPacketsMetricName, "Returns destination stuck packets for a channel.", []string{ - "channel_id", + "src_channel_id", + "dst_channel_id", "src_chain_id", "dst_chain_id", "status", @@ -129,6 +132,7 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { if err != nil { status = errorStatus + fmt.Println("FOOOOBAAAAR") log.Error(err.Error()) } @@ -138,7 +142,8 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(sp.StuckPackets.Total), []string{ - sp.Name, + sp.Source, + sp.Destination, (*cc.RPCs)[path.Chain1.ChainName].ChainID, (*cc.RPCs)[path.Chain2.ChainName].ChainID, status, @@ -150,7 +155,8 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(sp.StuckPackets.Source), []string{ - sp.Name, + sp.Source, + sp.Destination, (*cc.RPCs)[path.Chain1.ChainName].ChainID, (*cc.RPCs)[path.Chain2.ChainName].ChainID, status, @@ -162,7 +168,8 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(sp.StuckPackets.Destination), []string{ - sp.Name, + sp.Source, + sp.Destination, (*cc.RPCs)[path.Chain1.ChainName].ChainID, (*cc.RPCs)[path.Chain2.ChainName].ChainID, status, diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index a9ea576..33e43fd 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -28,7 +28,8 @@ type ChannelsInfo struct { } type Channel struct { - Name string + Source string + Destination string StuckPackets struct { Source int Destination int @@ -133,7 +134,8 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel var channel Channel - channel.Name = c.Chain1.ChannelID + channel.Source = c.Chain1.ChannelID + channel.Destination = c.Chain2.ChannelID switch c.Ordering { case "none": From bceaf105b205e5f21b9b45497420de47e86e6947 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Tue, 31 Oct 2023 10:00:40 +0200 Subject: [PATCH 10/13] fix: have status=error on bad RPC nodes --- pkg/collector/collector.go | 82 +++++++++++++++++++------------------- pkg/ibc/ibc.go | 11 ++++- 2 files changed, 52 insertions(+), 41 deletions(-) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 7e5c1a4..192fb78 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -3,6 +3,7 @@ package collector import ( "fmt" "math/big" + "reflect" "sync" "github.com/cosmos/relayer/v2/relayer" @@ -132,49 +133,50 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { if err != nil { status = errorStatus - fmt.Println("FOOOOBAAAAR") log.Error(err.Error()) } - for _, sp := range stuckPackets.Channels { - ch <- prometheus.MustNewConstMetric( - channelStuckPackets, - prometheus.GaugeValue, - float64(sp.StuckPackets.Total), - []string{ - sp.Source, - sp.Destination, - (*cc.RPCs)[path.Chain1.ChainName].ChainID, - (*cc.RPCs)[path.Chain2.ChainName].ChainID, - status, - }..., - ) - - ch <- prometheus.MustNewConstMetric( - channelSrcStuckPackets, - prometheus.GaugeValue, - float64(sp.StuckPackets.Source), - []string{ - sp.Source, - sp.Destination, - (*cc.RPCs)[path.Chain1.ChainName].ChainID, - (*cc.RPCs)[path.Chain2.ChainName].ChainID, - status, - }..., - ) - - ch <- prometheus.MustNewConstMetric( - channelDstStuckPackets, - prometheus.GaugeValue, - float64(sp.StuckPackets.Destination), - []string{ - sp.Source, - sp.Destination, - (*cc.RPCs)[path.Chain1.ChainName].ChainID, - (*cc.RPCs)[path.Chain2.ChainName].ChainID, - status, - }..., - ) + if !reflect.DeepEqual(stuckPackets, ibc.ChannelsInfo{}) { + for _, sp := range stuckPackets.Channels { + ch <- prometheus.MustNewConstMetric( + channelStuckPackets, + prometheus.GaugeValue, + float64(sp.StuckPackets.Total), + []string{ + sp.Source, + sp.Destination, + (*cc.RPCs)[path.Chain1.ChainName].ChainID, + (*cc.RPCs)[path.Chain2.ChainName].ChainID, + status, + }..., + ) + + ch <- prometheus.MustNewConstMetric( + channelSrcStuckPackets, + prometheus.GaugeValue, + float64(sp.StuckPackets.Source), + []string{ + sp.Source, + sp.Destination, + (*cc.RPCs)[path.Chain1.ChainName].ChainID, + (*cc.RPCs)[path.Chain2.ChainName].ChainID, + status, + }..., + ) + + ch <- prometheus.MustNewConstMetric( + channelDstStuckPackets, + prometheus.GaugeValue, + float64(sp.StuckPackets.Destination), + []string{ + sp.Source, + sp.Destination, + (*cc.RPCs)[path.Chain1.ChainName].ChainID, + (*cc.RPCs)[path.Chain2.ChainName].ChainID, + status, + }..., + ) + } } }(p) } diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index 33e43fd..5889e97 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -24,6 +24,8 @@ type ClientsInfo struct { } type ChannelsInfo struct { + ChainA *relayer.Chain + ChainB *relayer.Chain Channels []Channel } @@ -126,7 +128,14 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel if _, _, err := relayer.QueryLatestHeights( ctx, chainA, chainB, ); err != nil { - return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdA) + for _, c := range ibc.Channels { + var channel Channel + channel.Source = c.Chain1.ChannelID + channel.Destination = c.Chain2.ChannelID + channelInfo.Channels = append(channelInfo.Channels, channel) + } + + return channelInfo, fmt.Errorf("Error: %w for %v", err, cdA) } for _, c := range ibc.Channels { From 321b778c92c5005cfa35e1154fc3722ab8411608 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Tue, 31 Oct 2023 11:06:34 +0200 Subject: [PATCH 11/13] feat: change metric logic to only have one --- pkg/collector/collector.go | 55 ++++++-------------------------------- pkg/ibc/ibc.go | 54 +++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 76 deletions(-) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 192fb78..0a8288b 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -16,13 +16,11 @@ import ( ) const ( - successStatus = "success" - errorStatus = "error" - clientExpiryMetricName = "cosmos_ibc_client_expiry" - walletBalanceMetricName = "cosmos_wallet_balance" - channelStuckPacketsMetricName = "cosmos_ibc_stuck_packets_total" - channelSrcStuckPacketsMetricName = "cosmos_ibc_stuck_packets_src" - channelDstStuckPacketsMetricName = "cosmos_ibc_stuck_packets_dst" + successStatus = "success" + errorStatus = "error" + clientExpiryMetricName = "cosmos_ibc_client_expiry" + walletBalanceMetricName = "cosmos_wallet_balance" + channelStuckPacketsMetricName = "cosmos_ibc_stuck_packets" ) var ( @@ -43,30 +41,6 @@ var ( }, nil, ) - channelSrcStuckPackets = prometheus.NewDesc( - channelSrcStuckPacketsMetricName, - "Returns source stuck packets for a channel.", - []string{ - "src_channel_id", - "dst_channel_id", - "src_chain_id", - "dst_chain_id", - "status", - }, - nil, - ) - channelDstStuckPackets = prometheus.NewDesc( - channelDstStuckPacketsMetricName, - "Returns destination stuck packets for a channel.", - []string{ - "src_channel_id", - "dst_channel_id", - "src_chain_id", - "dst_chain_id", - "status", - }, - nil, - ) walletBalance = prometheus.NewDesc( walletBalanceMetricName, "Returns wallet balance for an address on a chain.", @@ -141,19 +115,6 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( channelStuckPackets, prometheus.GaugeValue, - float64(sp.StuckPackets.Total), - []string{ - sp.Source, - sp.Destination, - (*cc.RPCs)[path.Chain1.ChainName].ChainID, - (*cc.RPCs)[path.Chain2.ChainName].ChainID, - status, - }..., - ) - - ch <- prometheus.MustNewConstMetric( - channelSrcStuckPackets, - prometheus.GaugeValue, float64(sp.StuckPackets.Source), []string{ sp.Source, @@ -165,14 +126,14 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { ) ch <- prometheus.MustNewConstMetric( - channelDstStuckPackets, + channelStuckPackets, prometheus.GaugeValue, float64(sp.StuckPackets.Destination), []string{ - sp.Source, sp.Destination, - (*cc.RPCs)[path.Chain1.ChainName].ChainID, + sp.Source, (*cc.RPCs)[path.Chain2.ChainName].ChainID, + (*cc.RPCs)[path.Chain1.ChainName].ChainID, status, }..., ) diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index 5889e97..b287c33 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -24,18 +24,18 @@ type ClientsInfo struct { } type ChannelsInfo struct { - ChainA *relayer.Chain - ChainB *relayer.Chain Channels []Channel } type Channel struct { - Source string - Destination string - StuckPackets struct { + Source string + Destination string + SourcePort string + DestinationPort string + Ordering string + StuckPackets struct { Source int Destination int - Total int } } @@ -95,8 +95,19 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel ctx := context.Background() channelInfo := ChannelsInfo{} + // Init channel data + for _, c := range ibc.Channels { + var channel Channel + channel.Source = c.Chain1.ChannelID + channel.Destination = c.Chain2.ChannelID + channel.SourcePort = c.Chain1.PortID + channel.DestinationPort = c.Chain2.PortID + channel.Ordering = c.Ordering + channelInfo.Channels = append(channelInfo.Channels, channel) + } + if (*rpcs)[ibc.Chain1.ChainName].ChainID == "" || (*rpcs)[ibc.Chain2.ChainName].ChainID == "" { - return ChannelsInfo{}, fmt.Errorf( + return channelInfo, fmt.Errorf( "Error: RPC data is missing, cannot retrieve channel data: %v", ibc.Channels, ) @@ -128,24 +139,12 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel if _, _, err := relayer.QueryLatestHeights( ctx, chainA, chainB, ); err != nil { - for _, c := range ibc.Channels { - var channel Channel - channel.Source = c.Chain1.ChannelID - channel.Destination = c.Chain2.ChannelID - channelInfo.Channels = append(channelInfo.Channels, channel) - } - return channelInfo, fmt.Errorf("Error: %w for %v", err, cdA) } - for _, c := range ibc.Channels { + for i, c := range channelInfo.Channels { var order chantypes.Order - var channel Channel - - channel.Source = c.Chain1.ChannelID - channel.Destination = c.Chain2.ChannelID - switch c.Ordering { case "none": order = chantypes.NONE @@ -159,20 +158,17 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel State: stateOpen, Ordering: order, Counterparty: chantypes.Counterparty{ - PortId: c.Chain2.PortID, - ChannelId: c.Chain2.ChannelID, + PortId: c.DestinationPort, + ChannelId: c.Destination, }, - PortId: c.Chain1.PortID, - ChannelId: c.Chain2.ChannelID, + PortId: c.SourcePort, + ChannelId: c.Source, } unrelayedSequences := relayer.UnrelayedSequences(ctx, chainA, chainB, &ch) - channel.StuckPackets.Total += len(unrelayedSequences.Src) + len(unrelayedSequences.Dst) - channel.StuckPackets.Source += len(unrelayedSequences.Src) - channel.StuckPackets.Destination += len(unrelayedSequences.Dst) - - channelInfo.Channels = append(channelInfo.Channels, channel) + channelInfo.Channels[i].StuckPackets.Source += len(unrelayedSequences.Src) + channelInfo.Channels[i].StuckPackets.Destination += len(unrelayedSequences.Dst) } return channelInfo, nil From cdc524e9b002e8a87fbebda35861cf703fbc9993 Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Tue, 31 Oct 2023 12:51:05 +0200 Subject: [PATCH 12/13] feat: add logger to PrepChain this removes the need of having two identical checks --- pkg/chain/chain.go | 7 ++++--- pkg/ibc/ibc.go | 7 ------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pkg/chain/chain.go b/pkg/chain/chain.go index 9f5e00b..980da4c 100644 --- a/pkg/chain/chain.go +++ b/pkg/chain/chain.go @@ -5,6 +5,7 @@ import ( "github.com/cosmos/relayer/v2/relayer" "github.com/cosmos/relayer/v2/relayer/chains/cosmos" + "go.uber.org/zap" ) const ( @@ -19,7 +20,7 @@ type Info struct { } func PrepChain(info Info) (*relayer.Chain, error) { - chain := relayer.Chain{} + logger := zap.NewNop() providerConfig := cosmos.CosmosProviderConfig{ ChainID: info.ChainID, Timeout: rpcTimeout, @@ -37,12 +38,12 @@ func PrepChain(info Info) (*relayer.Chain, error) { return nil, err } - chain.ChainProvider = provider + chain := relayer.NewChain(logger, provider, false) err = chain.SetPath(&relayer.PathEnd{ClientID: info.ClientID}) if err != nil { return nil, err } - return &chain, nil + return chain, nil } diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index b287c33..5369faa 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -135,13 +135,6 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdB) } - // test that RPC endpoints are working - if _, _, err := relayer.QueryLatestHeights( - ctx, chainA, chainB, - ); err != nil { - return channelInfo, fmt.Errorf("Error: %w for %v", err, cdA) - } - for i, c := range channelInfo.Channels { var order chantypes.Order From 1ddbb9170c76a9acd7b57db1b583600a7baa8cac Mon Sep 17 00:00:00 2001 From: Joonas Lehtimaki Date: Tue, 31 Oct 2023 13:08:49 +0200 Subject: [PATCH 13/13] refactor: readd the check and add separate statuscheck --- pkg/collector/collector.go | 4 ++++ pkg/ibc/ibc.go | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 0a8288b..5454ae9 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -80,6 +80,7 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { go func(path *relayer.IBCdata) { defer wg.Done() + // Client info ci, err := ibc.GetClientsInfo(path, cc.RPCs) status := successStatus @@ -103,6 +104,9 @@ func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) { []string{(*cc.RPCs)[path.Chain2.ChainName].ChainID, path.Chain2.ClientID, (*cc.RPCs)[path.Chain1.ChainName].ChainID, status}..., ) + // Stuck packets + status = successStatus + stuckPackets, err := ibc.GetChannelsInfo(path, cc.RPCs) if err != nil { status = errorStatus diff --git a/pkg/ibc/ibc.go b/pkg/ibc/ibc.go index 5369faa..b287c33 100644 --- a/pkg/ibc/ibc.go +++ b/pkg/ibc/ibc.go @@ -135,6 +135,13 @@ func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (Channel return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdB) } + // test that RPC endpoints are working + if _, _, err := relayer.QueryLatestHeights( + ctx, chainA, chainB, + ); err != nil { + return channelInfo, fmt.Errorf("Error: %w for %v", err, cdA) + } + for i, c := range channelInfo.Channels { var order chantypes.Order