-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add prometheus metrics extension
- Loading branch information
Showing
4 changed files
with
348 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
module github.com/exaring/hoglet/extensions/prometheus | ||
|
||
go 1.21.3 | ||
|
||
require ( | ||
github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec | ||
github.com/prometheus/client_golang v1.17.0 | ||
github.com/stretchr/testify v1.8.2 | ||
) | ||
|
||
require ( | ||
github.com/beorn7/perks v1.0.1 // indirect | ||
github.com/cespare/xxhash/v2 v2.2.0 // indirect | ||
github.com/davecgh/go-spew v1.1.1 // indirect | ||
github.com/golang/protobuf v1.5.3 // indirect | ||
github.com/kr/text v0.2.0 // indirect | ||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect | ||
github.com/pmezard/go-difflib v1.0.0 // indirect | ||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect | ||
github.com/prometheus/common v0.44.0 // indirect | ||
github.com/prometheus/procfs v0.11.1 // indirect | ||
golang.org/x/sync v0.5.0 // indirect | ||
golang.org/x/sys v0.11.0 // indirect | ||
google.golang.org/protobuf v1.31.0 // indirect | ||
gopkg.in/yaml.v3 v3.0.1 // indirect | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= | ||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= | ||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= | ||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= | ||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= | ||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= | ||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec h1:4ITVxT5N5odnIn5f8qJlFLHu5KnHl87vfiOIOVSGhuY= | ||
github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec/go.mod h1:05lKY5CjhK3UHc9hr6VZdU6PAdmXumJksCVPdD6yrEQ= | ||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | ||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= | ||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= | ||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= | ||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= | ||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= | ||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= | ||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= | ||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= | ||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= | ||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= | ||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= | ||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= | ||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= | ||
github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= | ||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= | ||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= | ||
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= | ||
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= | ||
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= | ||
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= | ||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= | ||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= | ||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | ||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= | ||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= | ||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= | ||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= | ||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= | ||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= | ||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= | ||
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= | ||
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= | ||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= | ||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= | ||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= | ||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= | ||
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= | ||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | ||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= | ||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= | ||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= | ||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= | ||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,140 @@ | ||
package hogprom | ||
|
||
import ( | ||
"context" | ||
"errors" | ||
"fmt" | ||
"strconv" | ||
"time" | ||
|
||
"github.com/exaring/hoglet" | ||
"github.com/prometheus/client_golang/prometheus" | ||
) | ||
|
||
const ( | ||
namespace = "hoglet" | ||
) | ||
|
||
// WithPrometheusMetrics returns a [hoglet.BreakerMiddleware] that registers prometheus metrics for the circuit breaker. | ||
// | ||
// ⚠️ Note: the provided name must be unique across all hoglet instances using the same registerer. | ||
func WithPrometheusMetrics(name string, reg prometheus.Registerer) hoglet.BreakerMiddleware { | ||
return func(next hoglet.ObserverFactory) (hoglet.ObserverFactory, error) { | ||
callDurations := prometheus.NewHistogramVec( | ||
prometheus.HistogramOpts{ | ||
Namespace: namespace, | ||
Subsystem: "circuit", | ||
Name: "call_durations_seconds", | ||
Help: "Call durations in seconds", | ||
ConstLabels: prometheus.Labels{ | ||
"circuit": name, | ||
}, | ||
}, | ||
[]string{"error"}, | ||
) | ||
|
||
droppedCalls := prometheus.NewCounterVec( | ||
prometheus.CounterOpts{ | ||
Namespace: namespace, | ||
Subsystem: "circuit", | ||
Name: "dropped_calls_total", | ||
Help: "Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function)", | ||
ConstLabels: prometheus.Labels{ | ||
"circuit": name, | ||
}, | ||
}, | ||
[]string{"cause"}, | ||
) | ||
|
||
inflightCalls := prometheus.NewGauge( | ||
prometheus.GaugeOpts{ | ||
Namespace: namespace, | ||
Subsystem: "circuit", | ||
Name: "inflight_calls_current", | ||
Help: "Current number of calls in-flight", | ||
ConstLabels: prometheus.Labels{ | ||
"circuit": name, | ||
}, | ||
}, | ||
) | ||
|
||
for _, c := range []prometheus.Collector{ | ||
callDurations, | ||
droppedCalls, | ||
inflightCalls, | ||
} { | ||
if err := reg.Register(c); err != nil { | ||
return nil, fmt.Errorf("hoglet: registering collector: %w", err) | ||
} | ||
} | ||
|
||
return &prometheusObserverFactory{ | ||
next: next, | ||
|
||
timesource: wallclock{}, | ||
|
||
callDurations: callDurations, | ||
droppedCalls: droppedCalls, | ||
inflightCalls: inflightCalls, | ||
}, nil | ||
} | ||
} | ||
|
||
type prometheusObserverFactory struct { | ||
next hoglet.ObserverFactory | ||
|
||
timesource timesource | ||
|
||
callDurations *prometheus.HistogramVec | ||
droppedCalls *prometheus.CounterVec | ||
inflightCalls prometheus.Gauge | ||
} | ||
|
||
func (pos *prometheusObserverFactory) ObserverForCall(ctx context.Context, state hoglet.State) (hoglet.Observer, error) { | ||
o, err := pos.next.ObserverForCall(ctx, state) | ||
if err != nil { | ||
pos.droppedCalls.WithLabelValues(errToCause(err)).Inc() | ||
return nil, err | ||
} | ||
start := pos.timesource.Now() | ||
pos.inflightCalls.Inc() | ||
return hoglet.ObserverFunc(func(b bool) { | ||
pos.callDurations.WithLabelValues(strconv.FormatBool(b)).Observe(pos.timesource.Since(start).Seconds()) | ||
pos.inflightCalls.Dec() | ||
o.Observe(b) | ||
}), nil | ||
} | ||
|
||
// errToCause converts known circuit errors to metric labels. | ||
func errToCause(err error) string { | ||
switch err { | ||
case hoglet.ErrCircuitOpen: | ||
return "circuit_open" | ||
case hoglet.ErrConcurrencyLimitReached: | ||
return "concurrency_limit" | ||
default: | ||
// leave the errors.Is check as last, since it carries a performance penalty | ||
if errors.Is(err, context.Canceled) { | ||
return "context_canceled" | ||
} else if errors.Is(err, context.DeadlineExceeded) { | ||
return "deadline_exceeded" | ||
} | ||
return "other" | ||
} | ||
} | ||
|
||
type timesource interface { | ||
Now() time.Time | ||
Since(time.Time) time.Duration | ||
} | ||
|
||
// wallclock wraps time.Now/time.Since to allow mocking | ||
type wallclock struct{} | ||
|
||
func (wallclock) Now() time.Time { | ||
return time.Now() | ||
} | ||
|
||
func (wallclock) Since(t time.Time) time.Duration { | ||
return time.Since(t) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,124 @@ | ||
package hogprom | ||
|
||
import ( | ||
"context" | ||
"strings" | ||
"testing" | ||
"time" | ||
|
||
"github.com/exaring/hoglet" | ||
"github.com/prometheus/client_golang/prometheus" | ||
"github.com/prometheus/client_golang/prometheus/testutil" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
type mockObserverFactory struct{} | ||
|
||
// ObserverForCall implements hoglet.ObserverFactory. | ||
func (*mockObserverFactory) ObserverForCall(_ context.Context, state hoglet.State) (hoglet.Observer, error) { | ||
// this simple factory abuses the state argument to directly control the result of the call | ||
switch state { | ||
case hoglet.StateClosed: | ||
return mockObserver{}, nil | ||
case hoglet.StateOpen: | ||
return nil, hoglet.ErrCircuitOpen | ||
default: | ||
panic("not implemented") | ||
} | ||
} | ||
|
||
type mockObserver struct{} | ||
|
||
func (mockObserver) Observe(bool) {} | ||
|
||
type mockTimesource struct { | ||
t time.Time | ||
} | ||
|
||
func (m mockTimesource) Now() time.Time { | ||
return m.t | ||
} | ||
|
||
func (m mockTimesource) Since(t time.Time) time.Duration { | ||
return m.t.Sub(t) | ||
} | ||
|
||
func TestWithPrometheusMetrics(t *testing.T) { | ||
reg := prometheus.NewPedanticRegistry() | ||
m := WithPrometheusMetrics("test", reg) | ||
of, err := m(&mockObserverFactory{}) | ||
require.NoError(t, err) | ||
|
||
mt := &mockTimesource{time.Now()} | ||
|
||
of.(*prometheusObserverFactory).timesource = mt | ||
|
||
inflightOut0 := `# HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight | ||
# TYPE hoglet_circuit_inflight_calls_current gauge | ||
hoglet_circuit_inflight_calls_current{circuit="test"} 0 | ||
` | ||
|
||
if err := testutil.GatherAndCompare(reg, strings.NewReader(inflightOut0)); err != nil { | ||
t.Fatal(err) | ||
} | ||
|
||
_, err = of.ObserverForCall(context.Background(), hoglet.StateOpen) | ||
require.ErrorIs(t, err, hoglet.ErrCircuitOpen) | ||
|
||
droppedOut1 := `# HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function) | ||
# TYPE hoglet_circuit_dropped_calls_total counter | ||
hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1 | ||
# HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight | ||
# TYPE hoglet_circuit_inflight_calls_current gauge | ||
hoglet_circuit_inflight_calls_current{circuit="test"} 0 | ||
` | ||
if err := testutil.GatherAndCompare(reg, strings.NewReader(droppedOut1)); err != nil { | ||
t.Fatal(err) | ||
} | ||
|
||
o, err := of.ObserverForCall(context.Background(), hoglet.StateClosed) | ||
require.NoError(t, err) | ||
|
||
inflightOut1 := `# HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function) | ||
# TYPE hoglet_circuit_dropped_calls_total counter | ||
hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1 | ||
# HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight | ||
# TYPE hoglet_circuit_inflight_calls_current gauge | ||
hoglet_circuit_inflight_calls_current{circuit="test"} 1 | ||
` | ||
if err := testutil.GatherAndCompare(reg, strings.NewReader(inflightOut1)); err != nil { | ||
t.Fatal(err) | ||
} | ||
|
||
mt.t = mt.t.Add(time.Second) // move the clock 1 second forward | ||
|
||
o.Observe(true) | ||
|
||
durationsOut1 := `# HELP hoglet_circuit_call_durations_seconds Call durations in seconds | ||
# TYPE hoglet_circuit_call_durations_seconds histogram | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.005"} 0 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.01"} 0 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.025"} 0 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.05"} 0 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.1"} 0 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.25"} 0 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.5"} 0 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="1"} 1 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="2.5"} 1 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="5"} 1 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="10"} 1 | ||
hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="+Inf"} 1 | ||
hoglet_circuit_call_durations_seconds_sum{circuit="test",error="true"} 1 | ||
hoglet_circuit_call_durations_seconds_count{circuit="test",error="true"} 1 | ||
# HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function) | ||
# TYPE hoglet_circuit_dropped_calls_total counter | ||
hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1 | ||
# HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight | ||
# TYPE hoglet_circuit_inflight_calls_current gauge | ||
hoglet_circuit_inflight_calls_current{circuit="test"} 0 | ||
` | ||
|
||
if err := testutil.GatherAndCompare(reg, strings.NewReader(durationsOut1)); err != nil { | ||
t.Fatal(err) | ||
} | ||
} |