diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee9cfcd445..ca1a1d6a997 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -202,6 +202,9 @@ Adding a new version? You'll need three changes: [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048) - Add support for Gateway API GRPCRoute and pass related Gateway API conformance test. [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) +- Added new metric for Prometheus called `ingress_controller_admission_count`. It's a counter and has two labels + `allowed` to indicate if the resource was allowed and `resource` to indicate the resource under admission. + [#6084](https://github.com/Kong/kubernetes-ingress-controller/issues/6084) ### Fixed diff --git a/internal/admission/handler.go b/internal/admission/handler.go index 0b3fd95dbd4..4c27b8de351 100644 --- a/internal/admission/handler.go +++ b/internal/admission/handler.go @@ -16,6 +16,7 @@ import ( ctrlref "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers/reference" "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/labels" + "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1" kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1" @@ -37,6 +38,8 @@ type RequestHandler struct { // referring the validated resource (Secret) to check the changes on // referred Secret will produce invalid configuration of the plugins. ReferenceIndexers ctrlref.CacheIndexers + // PromMetrics provides the Prometheus registry to record metrics + PromMetrics *metrics.CtrlFuncMetrics Logger logr.Logger } @@ -63,6 +66,14 @@ func (h RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + + h.PromMetrics.RecordAdmissionCount( + response.Allowed, + fmt.Sprintf( + "%s.%s/%s", + review.Request.Resource.Resource, review.Request.Resource.Group, review.Request.Resource.Version, + ), + ) review.Response = response if err := json.NewEncoder(w).Encode(&review); err != nil { diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 1fcdb1ca562..aa2fe0bbb9b 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -197,12 +197,13 @@ func NewKongClient( kongConfigBuilder KongConfigBuilder, cacheStores store.CacheStores, fallbackConfigGenerator FallbackConfigGenerator, + prometheusMetrics *metrics.CtrlFuncMetrics, ) (*KongClient, error) { c := &KongClient{ logger: logger, requestTimeout: timeout, diagnostic: diagnostic, - prometheusMetrics: metrics.NewCtrlFuncMetrics(), + prometheusMetrics: prometheusMetrics, cache: &cacheStores, kongConfig: kongConfig, eventRecorder: eventRecorder, diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index 84384f927e8..a6eabc3a077 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -939,6 +939,7 @@ func setupTestKongClient( configBuilder, store.NewCacheStores(), newMockFallbackConfigGenerator(), + metrics.NewCtrlFuncMetrics(), ) require.NoError(t, err) return kongClient @@ -1386,6 +1387,7 @@ func TestKongClient_FallbackConfiguration_SkipMakingRedundantSnapshot(t *testing configBuilder, originalCache, fallbackConfigGenerator, + metrics.NewCtrlFuncMetrics(), ) require.NoError(t, err) @@ -1451,6 +1453,7 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) { configBuilder, originalCache, fallbackConfigGenerator, + metrics.NewCtrlFuncMetrics(), ) require.NoError(t, err) diff --git a/internal/manager/run.go b/internal/manager/run.go index 8d4f1d9a11f..d77a0a87d75 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -37,6 +37,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/metadata" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/telemetry" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/utils/kongconfig" + "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/kubernetes/object/status" @@ -185,7 +186,8 @@ func Run( } setupLog.Info("Starting Admission Server") - if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer); err != nil { + promMetrics := metrics.NewCtrlFuncMetrics() + if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer, promMetrics); err != nil { return err } @@ -207,6 +209,7 @@ func Run( configTranslator, cache, fallbackConfigGenerator, + promMetrics, ) if err != nil { return fmt.Errorf("failed to initialize kong data-plane client: %w", err) diff --git a/internal/manager/setup.go b/internal/manager/setup.go index ec61cbae021..d0a33735a8a 100644 --- a/internal/manager/setup.go +++ b/internal/manager/setup.go @@ -34,6 +34,7 @@ import ( konnectLicense "github.com/kong/kubernetes-ingress-controller/v3/internal/konnect/license" "github.com/kong/kubernetes-ingress-controller/v3/internal/license" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/scheme" + "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/kubernetes/object/status" @@ -195,6 +196,7 @@ func setupAdmissionServer( logger logr.Logger, translatorFeatures translator.FeatureFlags, storer store.Storer, + promMetrics *metrics.CtrlFuncMetrics, ) error { admissionLogger := logger.WithName("admission-server") @@ -214,6 +216,7 @@ func setupAdmissionServer( storer, ), ReferenceIndexers: referenceIndexers, + PromMetrics: promMetrics, Logger: admissionLogger, }, admissionLogger) if err != nil { diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go index f007da15207..7c58b6effaa 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/prometheus.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "strconv" "sync" "time" @@ -23,6 +24,7 @@ type CtrlFuncMetrics struct { TranslationBrokenResources prometheus.Gauge ConfigPushDuration *prometheus.HistogramVec ConfigPushSuccessTime *prometheus.GaugeVec + AdmissionCount *prometheus.CounterVec // Fallback config push metrics. FallbackTranslationCount *prometheus.CounterVec @@ -77,6 +79,17 @@ const ( DataplaneKey string = "dataplane" ) +// Regular config push metrics names. +const ( + // AllowedKey defines the key of the metric label indicating admission was allowed. + AllowedKey string = "allowed" +) + +const ( + // AdmissionResourceKey defines the name of the metric label indicating which dataplane this time series is relevant for. + AdmissionResourceKey string = "resource" +) + // Regular config push metrics names. const ( MetricNameConfigPushCount = "ingress_controller_configuration_push_count" @@ -85,6 +98,7 @@ const ( MetricNameTranslationCount = "ingress_controller_translation_count" MetricNameTranslationBrokenResources = "ingress_controller_translation_broken_resource_count" MetricNameConfigPushDuration = "ingress_controller_configuration_push_duration_milliseconds" + MetricNameAdmissionCount = "ingress_controller_admission_count" ) // Fallback config push metrics names. @@ -189,6 +203,20 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics { []string{DataplaneKey}, ) + controllerMetrics.AdmissionCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricNameAdmissionCount, + Help: fmt.Sprintf( + "Count of admissions processed by Kong. "+ + "`%s` describes whether an admission was allowed. "+ + "`%s` describes the resource under admission. ", + AllowedKey, + AdmissionResourceKey, + ), + }, + []string{AllowedKey, AdmissionResourceKey}, + ) + controllerMetrics.FallbackTranslationCount = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: MetricNameFallbackTranslationCount, @@ -302,6 +330,7 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics { controllerMetrics.TranslationBrokenResources, controllerMetrics.ConfigPushDuration, controllerMetrics.ConfigPushSuccessTime, + controllerMetrics.AdmissionCount, controllerMetrics.FallbackTranslationBrokenResources, controllerMetrics.FallbackTranslationCount, controllerMetrics.FallbackConfigPushCount, @@ -356,6 +385,13 @@ func (c *CtrlFuncMetrics) RecordTranslationBrokenResources(count int) { c.TranslationBrokenResources.Set(float64(count)) } +func (c *CtrlFuncMetrics) RecordAdmissionCount(allowed bool, resource string) { + c.ConfigPushCount.With(prometheus.Labels{ + AllowedKey: strconv.FormatBool(allowed), + AdmissionResourceKey: resource, + }).Inc() +} + // RecordFallbackTranslationFailure records a failed fallback configuration translation. func (c *CtrlFuncMetrics) RecordFallbackTranslationFailure() { c.FallbackTranslationCount.With(prometheus.Labels{ diff --git a/test/envtest/metrics_envtest_test.go b/test/envtest/metrics_envtest_test.go index e2276ac6ed1..30a2f485266 100644 --- a/test/envtest/metrics_envtest_test.go +++ b/test/envtest/metrics_envtest_test.go @@ -70,6 +70,7 @@ func TestMetricsAreServed(t *testing.T) { metrics.MetricNameTranslationBrokenResources, metrics.MetricNameConfigPushDuration, metrics.MetricNameConfigPushSuccessTime, + metrics.MetricNameAdmissionCount, }, }, }