From 654bc59c433347da0925f9e4dc061e931db9461e Mon Sep 17 00:00:00 2001 From: David Weber Date: Sun, 26 May 2024 02:00:52 +0200 Subject: [PATCH] feat: add counter for admissions Signed-off-by: David Weber --- CHANGELOG.md | 3 +++ internal/admission/handler.go | 11 ++++++++ internal/dataplane/kong_client.go | 3 ++- internal/dataplane/kong_client_test.go | 3 +++ internal/manager/run.go | 5 +++- internal/manager/setup.go | 3 +++ internal/metrics/prometheus.go | 37 ++++++++++++++++++++++++++ test/envtest/metrics_envtest_test.go | 1 + 8 files changed, 64 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa4f1c29f12..7ec8c35b185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,6 +159,9 @@ Adding a new version? You'll need three changes: [#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965) - Fallback configuration no longer omits licenses and vaults. [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048) +- Added new metric for Prometheus called `ingress_controller_admission_count`. It's a counter and has two labels + `allowed` to indicate if the resource was allowed and `resource` to indicate the resource under admission. + [#6084](https://github.com/Kong/kubernetes-ingress-controller/issues/6084) ### Fixed diff --git a/internal/admission/handler.go b/internal/admission/handler.go index 0b3fd95dbd4..4c27b8de351 100644 --- a/internal/admission/handler.go +++ b/internal/admission/handler.go @@ -16,6 +16,7 @@ import ( ctrlref "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers/reference" "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/labels" + "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1" kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1" @@ -37,6 +38,8 @@ type RequestHandler struct { // referring the validated resource (Secret) to check the changes on // referred Secret will produce invalid configuration of the plugins. ReferenceIndexers ctrlref.CacheIndexers + // PromMetrics provides the Prometheus registry to record metrics + PromMetrics *metrics.CtrlFuncMetrics Logger logr.Logger } @@ -63,6 +66,14 @@ func (h RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + + h.PromMetrics.RecordAdmissionCount( + response.Allowed, + fmt.Sprintf( + "%s.%s/%s", + review.Request.Resource.Resource, review.Request.Resource.Group, review.Request.Resource.Version, + ), + ) review.Response = response if err := json.NewEncoder(w).Encode(&review); err != nil { diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 0d738b1027f..eb816be18b9 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -169,12 +169,13 @@ func NewKongClient( kongConfigBuilder KongConfigBuilder, cacheStores store.CacheStores, fallbackConfigGenerator FallbackConfigGenerator, + prometheusMetrics *metrics.CtrlFuncMetrics, ) (*KongClient, error) { c := &KongClient{ logger: logger, requestTimeout: timeout, diagnostic: diagnostic, - prometheusMetrics: metrics.NewCtrlFuncMetrics(), + prometheusMetrics: prometheusMetrics, cache: &cacheStores, kongConfig: kongConfig, eventRecorder: eventRecorder, diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index f0364146702..4481ad7d1ce 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -733,6 +733,7 @@ func setupTestKongClient( configBuilder, store.NewCacheStores(), newMockFallbackConfigGenerator(), + metrics.NewCtrlFuncMetrics(), ) require.NoError(t, err) return kongClient @@ -1002,6 +1003,7 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { configBuilder, originalCache, fallbackConfigGenerator, + metrics.NewCtrlFuncMetrics(), ) require.NoError(t, err) @@ -1114,6 +1116,7 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) { configBuilder, originalCache, fallbackConfigGenerator, + metrics.NewCtrlFuncMetrics(), ) require.NoError(t, err) diff --git a/internal/manager/run.go b/internal/manager/run.go index a56b4ba1298..fc5308541c6 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -36,6 +36,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/metadata" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/telemetry" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/utils/kongconfig" + "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/kubernetes/object/status" @@ -185,7 +186,8 @@ func Run( } setupLog.Info("Starting Admission Server") - if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer); err != nil { + promMetrics := metrics.NewCtrlFuncMetrics() + if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer, promMetrics); err != nil { return err } @@ -207,6 +209,7 @@ func Run( configTranslator, cache, fallbackConfigGenerator, + promMetrics, ) if err != nil { return fmt.Errorf("failed to initialize kong data-plane client: %w", err) diff --git a/internal/manager/setup.go b/internal/manager/setup.go index ec61cbae021..d0a33735a8a 100644 --- a/internal/manager/setup.go +++ b/internal/manager/setup.go @@ -34,6 +34,7 @@ import ( konnectLicense "github.com/kong/kubernetes-ingress-controller/v3/internal/konnect/license" "github.com/kong/kubernetes-ingress-controller/v3/internal/license" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/scheme" + "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/kubernetes/object/status" @@ -195,6 +196,7 @@ func setupAdmissionServer( logger logr.Logger, translatorFeatures translator.FeatureFlags, storer store.Storer, + promMetrics *metrics.CtrlFuncMetrics, ) error { admissionLogger := logger.WithName("admission-server") @@ -214,6 +216,7 @@ func setupAdmissionServer( storer, ), ReferenceIndexers: referenceIndexers, + PromMetrics: promMetrics, Logger: admissionLogger, }, admissionLogger) if err != nil { diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go index 11194bd93ba..07faf9ac3a0 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/prometheus.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "strconv" "sync" "time" @@ -27,6 +28,8 @@ type CtrlFuncMetrics struct { ConfigPushDuration *prometheus.HistogramVec ConfigPushSuccessTime *prometheus.GaugeVec + + AdmissionCount *prometheus.CounterVec } const ( @@ -70,6 +73,16 @@ const ( DataplaneKey string = "dataplane" ) +const ( + // AllowedKey defines the key of the metric label indicating admission was allowed. + AllowedKey string = "allowed" +) + +const ( + // AdmissionResourceKey defines the name of the metric label indicating which dataplane this time series is relevant for. + AdmissionResourceKey string = "resource" +) + const ( MetricNameConfigPushCount = "ingress_controller_configuration_push_count" MetricNameConfigPushBrokenResources = "ingress_controller_configuration_push_broken_resource_count" @@ -77,6 +90,7 @@ const ( MetricNameTranslationCount = "ingress_controller_translation_count" MetricNameTranslationBrokenResources = "ingress_controller_translation_broken_resource_count" MetricNameConfigPushDuration = "ingress_controller_configuration_push_duration_milliseconds" + MetricNameAdmissionCount = "ingress_controller_admission_count" ) var _lock sync.Mutex @@ -168,12 +182,27 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics { []string{DataplaneKey}, ) + controllerMetrics.AdmissionCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricNameAdmissionCount, + Help: fmt.Sprintf( + "Count of admissions processed by Kong. "+ + "`%s` describes whether an admission was allowed. "+ + "`%s` describes the resource under admission. ", + AllowedKey, + AdmissionResourceKey, + ), + }, + []string{AllowedKey, AdmissionResourceKey}, + ) + metrics.Registry.Unregister(controllerMetrics.ConfigPushCount) metrics.Registry.Unregister(controllerMetrics.ConfigPushBrokenResources) metrics.Registry.Unregister(controllerMetrics.TranslationCount) metrics.Registry.Unregister(controllerMetrics.TranslationBrokenResources) metrics.Registry.Unregister(controllerMetrics.ConfigPushDuration) metrics.Registry.Unregister(controllerMetrics.ConfigPushSuccessTime) + metrics.Registry.Unregister(controllerMetrics.AdmissionCount) metrics.Registry.MustRegister( controllerMetrics.ConfigPushCount, @@ -182,6 +211,7 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics { controllerMetrics.TranslationBrokenResources, controllerMetrics.ConfigPushDuration, controllerMetrics.ConfigPushSuccessTime, + controllerMetrics.AdmissionCount, ) return controllerMetrics @@ -223,6 +253,13 @@ func (c *CtrlFuncMetrics) RecordTranslationBrokenResources(count int) { c.TranslationBrokenResources.Set(float64(count)) } +func (c *CtrlFuncMetrics) RecordAdmissionCount(allowed bool, resource string) { + c.ConfigPushCount.With(prometheus.Labels{ + AllowedKey: strconv.FormatBool(allowed), + AdmissionResourceKey: resource, + }).Inc() +} + type recordOption func(prometheus.Labels) prometheus.Labels func withError(err error) recordOption { diff --git a/test/envtest/metrics_envtest_test.go b/test/envtest/metrics_envtest_test.go index af0df4f2f53..dacf99cf64d 100644 --- a/test/envtest/metrics_envtest_test.go +++ b/test/envtest/metrics_envtest_test.go @@ -42,6 +42,7 @@ func TestMetricsAreServed(t *testing.T) { metrics.MetricNameTranslationBrokenResources, metrics.MetricNameConfigPushDuration, metrics.MetricNameConfigPushSuccessTime, + metrics.MetricNameAdmissionCount, } metricsURL := fmt.Sprintf("http://%s/metrics", cfg.MetricsAddr)