Merge pull request #27 from vinted/add_tweaks

gate: add metric for wait

storage/remote/read_handler.go

@@ -47,7 +47,7 @@ func NewReadHandler(logger log.Logger, r prometheus.Registerer, queryable storag
 		queryable:                 queryable,
 		config:                    config,
 		remoteReadSampleLimit:     remoteReadSampleLimit,
-		remoteReadGate:            gate.New(remoteReadConcurrencyLimit),
+		remoteReadGate:            gate.New(remoteReadConcurrencyLimit, "read_handler", r),
 		remoteReadMaxBytesInFrame: remoteReadMaxBytesInFrame,
 
 		queries: prometheus.NewGauge(prometheus.GaugeOpts{

util/gate/gate.go

@@ -13,23 +13,42 @@
 
 package gate
 
-import "context"
+import (
+	"context"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
 
 // A Gate controls the maximum number of concurrently running and waiting queries.
 type Gate struct {
-	ch chan struct{}
+	ch           chan struct{}
+	waitDuration prometheus.Histogram
 }
 
 // New returns a query gate that limits the number of queries
 // being concurrently executed.
-func New(length int) *Gate {
+func New(length int, name string, r prometheus.Registerer) *Gate {
 	return &Gate{
 		ch: make(chan struct{}, length),
+		waitDuration: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
+			Namespace:   "prometheus",
+			Subsystem:   "api",
+			Help:        "How long was the wait at the gate before the query execution",
+			Buckets:     prometheus.ExponentialBuckets(0.1, 2, 10),
+			Name:        "gate_wait_duration_seconds",
+			ConstLabels: prometheus.Labels{"name": name},
+		}),
 	}
 }
 
 // Start blocks until the gate has a free spot or the context is done.
 func (g *Gate) Start(ctx context.Context) error {
+	startTime := time.Now()
+	defer func() {
+		g.waitDuration.Observe(float64(time.Since(startTime).Seconds()))
+	}()
 	select {
 	case <-ctx.Done():
 		return ctx.Err()