From 7edd9355217166bf6c6b47b53dd77b2f8bd43e4d Mon Sep 17 00:00:00 2001 From: David Nix Date: Fri, 12 May 2023 08:17:02 -0600 Subject: [PATCH 1/7] Rename to ReferenceAPI --- metrics/reference_rpc.go | 22 +++++++++++----------- metrics/reference_rpc_test.go | 12 ++++-------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/metrics/reference_rpc.go b/metrics/reference_rpc.go index bedabc1..aed8b62 100644 --- a/metrics/reference_rpc.go +++ b/metrics/reference_rpc.go @@ -6,30 +6,30 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// ReferenceRPC records metrics for external RPC calls. -type ReferenceRPC struct { +// ReferenceAPI records metrics for external http calls. +type ReferenceAPI struct { errorCounter *prometheus.CounterVec // TODO(nix): Count requests and histogram of latency. } -func NewReferenceRPC() *ReferenceRPC { - const subsystem = "reference_rpc" - return &ReferenceRPC{ +func NewHTTPRequest() *ReferenceAPI { + const subsystem = "reference_api" + return &ReferenceAPI{ errorCounter: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: prometheus.BuildFQName(Namespace, subsystem, "error_count"), - Help: "Number of errors encountered while making external RPC, API, or GRPC calls.", + Name: prometheus.BuildFQName(namespace, subsystem, "error_count"), + Help: "Number of errors encountered while making external calls to an API to gather reference data.", }, - []string{"type", "host", "reason"}, + []string{"host", "reason"}, ), } } -func (c ReferenceRPC) IncClientError(rpcType string, host url.URL, reason string) { - c.errorCounter.WithLabelValues(rpcType, host.Hostname(), reason).Inc() +func (c ReferenceAPI) IncAPIError(host url.URL, reason string) { + c.errorCounter.WithLabelValues(host.Hostname(), reason).Inc() } -func (c ReferenceRPC) Metrics() []prometheus.Collector { +func (c ReferenceAPI) Metrics() []prometheus.Collector { return []prometheus.Collector{ c.errorCounter, } diff --git a/metrics/reference_rpc_test.go b/metrics/reference_rpc_test.go index 3df2f2a..d4bc8c3 100644 --- a/metrics/reference_rpc_test.go +++ b/metrics/reference_rpc_test.go @@ -3,31 +3,27 @@ package metrics import ( "net/http/httptest" "net/url" - "strings" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" ) -func TestReferenceRPC_IncClientError(t *testing.T) { +func TestReferenceAPI_IncAPIError(t *testing.T) { t.Parallel() reg := prometheus.NewRegistry() - metrics := NewReferenceRPC() + metrics := NewHTTPRequest() reg.MustRegister(metrics.Metrics()[0]) u, err := url.Parse("http://test.example/should/not/be/used") require.NoError(t, err) - metrics.IncClientError("cosmos-lcd", *u, "timeout") + metrics.IncAPIError(*u, "timeout") h := metricsHandler(reg) r := httptest.NewRecorder() h.ServeHTTP(r, stubRequest) - const want = `# HELP sl_exporter_reference_rpc_error_count Number of errors encountered while making external RPC, API, or GRPC calls. -# TYPE sl_exporter_reference_rpc_error_count counter -sl_exporter_reference_rpc_error_count{host="test.example",reason="timeout",type="cosmos-lcd"} 1` - require.Equal(t, strings.TrimSpace(want), strings.TrimSpace(r.Body.String())) + require.Contains(t, r.Body.String(), `sl_exporter_reference_api_error_count{host="test.example",reason="timeout"} 1`) } From f293b44c8ffb567758003c924242ff1bb3f15bd2 Mon Sep 17 00:00:00 2001 From: David Nix Date: Fri, 12 May 2023 08:19:40 -0600 Subject: [PATCH 2/7] Refactor fallback client for new interface --- cmd/root.go | 4 ++-- metrics/fallback_client.go | 12 +++++------- metrics/fallback_client_test.go | 17 +++++++---------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 9ea2f97..6dc913d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -60,7 +60,7 @@ func Execute() { registry.MustRegister(metrics.BuildStatic(cfg.Static.Gauges)...) // Register reference rpc metrics - refMets := metrics.NewReferenceRPC() + refMets := metrics.NewHTTPRequest() registry.MustRegister(refMets.Metrics()...) // Register cosmos chain metrics @@ -125,7 +125,7 @@ func logFatal(msg string, err error) { os.Exit(1) } -func buildCosmosJobs(cosmosMets *metrics.Cosmos, refMets *metrics.ReferenceRPC, cfg Config) (jobs []metrics.Job) { +func buildCosmosJobs(cosmosMets *metrics.Cosmos, refMets *metrics.ReferenceAPI, cfg Config) (jobs []metrics.Job) { // TODO(nix): Need different rest clients per chain. This hack prevents > 1 chain. var urls []url.URL for _, rest := range cfg.Cosmos[0].Rest { diff --git a/metrics/fallback_client.go b/metrics/fallback_client.go index 92a6fae..127111a 100644 --- a/metrics/fallback_client.go +++ b/metrics/fallback_client.go @@ -16,15 +16,14 @@ type FallbackClient struct { httpDo func(req *http.Request) (*http.Response, error) log *slog.Logger metrics ClientMetrics - rpcType string } type ClientMetrics interface { - IncClientError(rpcType string, host url.URL, reason string) + IncAPIError(host url.URL, reason string) // TODO(nix): Metrics for request counts. Latency histogram. } -func NewFallbackClient(client *http.Client, metrics ClientMetrics, rpcType string, hosts []url.URL) *FallbackClient { +func NewFallbackClient(client *http.Client, metrics ClientMetrics, hosts []url.URL) *FallbackClient { if len(hosts) == 0 { panic("no hosts provided") } @@ -33,7 +32,6 @@ func NewFallbackClient(client *http.Client, metrics ClientMetrics, rpcType strin httpDo: client.Do, log: slog.Default(), metrics: metrics, - rpcType: rpcType, } } @@ -41,7 +39,7 @@ const unknownErrReason = "unknown" func (c FallbackClient) Get(ctx context.Context, path string) (*http.Response, error) { doGet := func(host url.URL) (*http.Response, error) { - log := c.log.With("host", host.Hostname(), "path", path, "rpc", c.rpcType) + log := c.log.With("host", host.Hostname(), "path", path) host.Path = path req, err := http.NewRequestWithContext(ctx, http.MethodGet, host.String(), nil) if err != nil { @@ -58,7 +56,7 @@ func (c FallbackClient) Get(ctx context.Context, path string) (*http.Response, e if resp.StatusCode < 200 || resp.StatusCode >= 300 { _ = resp.Body.Close() log.Error("Response returned bad status code", "status", resp.StatusCode) - c.metrics.IncClientError(c.rpcType, host, strconv.Itoa(resp.StatusCode)) + c.metrics.IncAPIError(host, strconv.Itoa(resp.StatusCode)) return nil, fmt.Errorf("%s: bad status code %d", req.URL, resp.StatusCode) } return resp, nil @@ -85,5 +83,5 @@ func (c FallbackClient) recordErrMetric(host url.URL, err error) { // Do not record when the process is shutting down. return } - c.metrics.IncClientError(c.rpcType, host, reason) + c.metrics.IncAPIError(host, reason) } diff --git a/metrics/fallback_client_test.go b/metrics/fallback_client_test.go index 1d8b0f7..dbb921e 100644 --- a/metrics/fallback_client_test.go +++ b/metrics/fallback_client_test.go @@ -17,14 +17,12 @@ import ( type mockClientMetrics struct { IncClientErrCalls int - GotRPCType string GotHost url.URL GotErrMsg string } -func (m *mockClientMetrics) IncClientError(rpcType string, host url.URL, errMsg string) { +func (m *mockClientMetrics) IncAPIError(host url.URL, errMsg string) { m.IncClientErrCalls++ - m.GotRPCType = rpcType m.GotHost = host m.GotErrMsg = errMsg } @@ -41,7 +39,7 @@ func TestFallbackClient_Get(t *testing.T) { ctx := context.WithValue(context.Background(), dummy("test"), dummy("test")) t.Run("happy path", func(t *testing.T) { - client := NewFallbackClient(&http.Client{}, nil, "test", urls) + client := NewFallbackClient(&http.Client{}, nil, urls) client.log = nopLogger require.NotNil(t, client.httpDo) @@ -65,7 +63,7 @@ func TestFallbackClient_Get(t *testing.T) { t.Run("fallback on error", func(t *testing.T) { var metrics mockClientMetrics - client := NewFallbackClient(nil, &metrics, "test", urls) + client := NewFallbackClient(nil, &metrics, urls) client.log = nopLogger var callCount int @@ -91,7 +89,7 @@ func TestFallbackClient_Get(t *testing.T) { t.Run("fallback on bad status code", func(t *testing.T) { var metrics mockClientMetrics - client := NewFallbackClient(nil, &metrics, "test", urls) + client := NewFallbackClient(nil, &metrics, urls) client.log = nopLogger var callCount int @@ -119,7 +117,7 @@ func TestFallbackClient_Get(t *testing.T) { t.Run("all errors", func(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) var metrics mockClientMetrics - client := NewFallbackClient(nil, &metrics, "test", urls) + client := NewFallbackClient(nil, &metrics, urls) client.log = nopLogger var callCount int @@ -154,7 +152,7 @@ func TestFallbackClient_Get(t *testing.T) { {nil, &http.Response{StatusCode: http.StatusNotFound}, "404"}, } { var metrics mockClientMetrics - client := NewFallbackClient(nil, &metrics, "test", []url.URL{{Host: "error.example.com"}}) + client := NewFallbackClient(nil, &metrics, []url.URL{{Host: "error.example.com"}}) client.log = nopLogger client.httpDo = func(req *http.Request) (*http.Response, error) { @@ -167,7 +165,6 @@ func TestFallbackClient_Get(t *testing.T) { //nolint _, _ = client.Get(ctx, "") - require.Equal(t, "test", metrics.GotRPCType, tt) require.Equal(t, "error.example.com", metrics.GotHost.Hostname(), tt) require.Equal(t, tt.WantMsg, metrics.GotErrMsg, tt) } @@ -175,7 +172,7 @@ func TestFallbackClient_Get(t *testing.T) { t.Run("context canceled error", func(t *testing.T) { var metrics mockClientMetrics - client := NewFallbackClient(nil, &metrics, "test", []url.URL{{Host: "error.example.com"}}) + client := NewFallbackClient(nil, &metrics, []url.URL{{Host: "error.example.com"}}) client.log = nopLogger client.httpDo = func(req *http.Request) (*http.Response, error) { From fafdf4feeb92b6ae7d63e065b4ad94fae1e45afb Mon Sep 17 00:00:00 2001 From: David Nix Date: Fri, 12 May 2023 08:21:27 -0600 Subject: [PATCH 3/7] Remove rpc type from main --- cmd/root.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6dc913d..9ce660b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -136,8 +136,7 @@ func buildCosmosJobs(cosmosMets *metrics.Cosmos, refMets *metrics.ReferenceAPI, urls = append(urls, *u) } - const rpcType = "cosmos" - restClient := cosmos.NewRestClient(metrics.NewFallbackClient(httpClient, refMets, rpcType, urls)) + restClient := cosmos.NewRestClient(metrics.NewFallbackClient(httpClient, refMets, urls)) restJobs := cosmos.BuildRestJobs(cosmosMets, restClient, cfg.Cosmos) jobs = append(jobs, toJobs(restJobs)...) From 36397ead294504fad498d29e4c8b1a5363553898 Mon Sep 17 00:00:00 2001 From: David Nix Date: Fri, 12 May 2023 08:23:02 -0600 Subject: [PATCH 4/7] rename file --- metrics/{reference_rpc.go => reference_api.go} | 0 metrics/{reference_rpc_test.go => reference_api_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename metrics/{reference_rpc.go => reference_api.go} (100%) rename metrics/{reference_rpc_test.go => reference_api_test.go} (100%) diff --git a/metrics/reference_rpc.go b/metrics/reference_api.go similarity index 100% rename from metrics/reference_rpc.go rename to metrics/reference_api.go diff --git a/metrics/reference_rpc_test.go b/metrics/reference_api_test.go similarity index 100% rename from metrics/reference_rpc_test.go rename to metrics/reference_api_test.go From 8801abea5a05437e2ec228590200ce945cfd1ed3 Mon Sep 17 00:00:00 2001 From: David Nix Date: Fri, 12 May 2023 08:27:45 -0600 Subject: [PATCH 5/7] Move subsystem to common place --- metrics/metrics.go | 5 ++++- metrics/reference_api.go | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 122ec5b..d8a3b1a 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,8 +1,11 @@ package metrics const ( - namespace = "sl_exporter" + namespace = "sl_exporter" + + // Subsystems staticSubsystem = "static" cosmosSubsystem = "cosmos" cosmosValSubsystem = cosmosSubsystem + "_val" + refAPISubsystem = "reference_api" ) diff --git a/metrics/reference_api.go b/metrics/reference_api.go index aed8b62..17718a3 100644 --- a/metrics/reference_api.go +++ b/metrics/reference_api.go @@ -13,11 +13,10 @@ type ReferenceAPI struct { } func NewHTTPRequest() *ReferenceAPI { - const subsystem = "reference_api" return &ReferenceAPI{ errorCounter: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: prometheus.BuildFQName(namespace, subsystem, "error_count"), + Name: prometheus.BuildFQName(namespace, refAPISubsystem, "error_count"), Help: "Number of errors encountered while making external calls to an API to gather reference data.", }, []string{"host", "reason"}, From 09c1593c0097082ffc1ea4cd6cfda4599f70c90b Mon Sep 17 00:00:00 2001 From: David Nix Date: Fri, 12 May 2023 08:28:39 -0600 Subject: [PATCH 6/7] Add http method log field --- metrics/fallback_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/fallback_client.go b/metrics/fallback_client.go index 127111a..cb5fe1b 100644 --- a/metrics/fallback_client.go +++ b/metrics/fallback_client.go @@ -39,7 +39,7 @@ const unknownErrReason = "unknown" func (c FallbackClient) Get(ctx context.Context, path string) (*http.Response, error) { doGet := func(host url.URL) (*http.Response, error) { - log := c.log.With("host", host.Hostname(), "path", path) + log := c.log.With("host", host.Hostname(), "path", path, "method", http.MethodGet) host.Path = path req, err := http.NewRequestWithContext(ctx, http.MethodGet, host.String(), nil) if err != nil { From 16755cfb53066c6ab38f941232bfbaaa8c4a0cc2 Mon Sep 17 00:00:00 2001 From: David Nix Date: Fri, 12 May 2023 12:20:28 -0600 Subject: [PATCH 7/7] Update metric name to total --- metrics/reference_api.go | 2 +- metrics/reference_api_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/reference_api.go b/metrics/reference_api.go index 17718a3..cff447b 100644 --- a/metrics/reference_api.go +++ b/metrics/reference_api.go @@ -16,7 +16,7 @@ func NewHTTPRequest() *ReferenceAPI { return &ReferenceAPI{ errorCounter: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: prometheus.BuildFQName(namespace, refAPISubsystem, "error_count"), + Name: prometheus.BuildFQName(namespace, refAPISubsystem, "error_total"), Help: "Number of errors encountered while making external calls to an API to gather reference data.", }, []string{"host", "reason"}, diff --git a/metrics/reference_api_test.go b/metrics/reference_api_test.go index d4bc8c3..2347425 100644 --- a/metrics/reference_api_test.go +++ b/metrics/reference_api_test.go @@ -25,5 +25,5 @@ func TestReferenceAPI_IncAPIError(t *testing.T) { r := httptest.NewRecorder() h.ServeHTTP(r, stubRequest) - require.Contains(t, r.Body.String(), `sl_exporter_reference_api_error_count{host="test.example",reason="timeout"} 1`) + require.Contains(t, r.Body.String(), `sl_exporter_reference_api_error_total{host="test.example",reason="timeout"} 1`) }