Skip to content

Commit

Permalink
feat: add counter for admissions
Browse files Browse the repository at this point in the history
Signed-off-by: David Weber <[email protected]>
  • Loading branch information
dweber019 committed May 27, 2024
1 parent 9b86378 commit fb7a9eb
Show file tree
Hide file tree
Showing 8 changed files with 65 additions and 4 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ Adding a new version? You'll need three changes:
[#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965)
- Fallback configuration no longer omits licenses and vaults.
[#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048)
- Added new metric for Prometheus called `ingress_controller_admission_count`. It's a counter and has two labels
`allowed` to indicate if the resource was allowed and `resource` to indicate the resource under admission.
[#6084](https://github.com/Kong/kubernetes-ingress-controller/issues/6084)

### Fixed

Expand Down
14 changes: 12 additions & 2 deletions internal/admission/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"

"github.com/go-logr/logr"
"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
admissionv1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"net/http"

"github.com/kong/kubernetes-ingress-controller/v3/internal/annotations"
ctrlref "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers/reference"
Expand All @@ -37,6 +37,8 @@ type RequestHandler struct {
// referring the validated resource (Secret) to check the changes on
// referred Secret will produce invalid configuration of the plugins.
ReferenceIndexers ctrlref.CacheIndexers
// PromMetrics provides the Prometheus registry to record metrics
PromMetrics *metrics.CtrlFuncMetrics

Logger logr.Logger
}
Expand All @@ -63,6 +65,14 @@ func (h RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

h.PromMetrics.RecordAdmissionCount(
response.Allowed,
fmt.Sprintf(
"%s.%s/%s",
review.Request.Resource.Resource, review.Request.Resource.Group, review.Request.Resource.Version,
),
)
review.Response = response

if err := json.NewEncoder(w).Encode(&review); err != nil {
Expand Down
3 changes: 2 additions & 1 deletion internal/dataplane/kong_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,12 +169,13 @@ func NewKongClient(
kongConfigBuilder KongConfigBuilder,
cacheStores store.CacheStores,
fallbackConfigGenerator FallbackConfigGenerator,
prometheusMetrics *metrics.CtrlFuncMetrics,
) (*KongClient, error) {
c := &KongClient{
logger: logger,
requestTimeout: timeout,
diagnostic: diagnostic,
prometheusMetrics: metrics.NewCtrlFuncMetrics(),
prometheusMetrics: prometheusMetrics,
cache: &cacheStores,
kongConfig: kongConfig,
eventRecorder: eventRecorder,
Expand Down
3 changes: 3 additions & 0 deletions internal/dataplane/kong_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -733,6 +733,7 @@ func setupTestKongClient(
configBuilder,
store.NewCacheStores(),
newMockFallbackConfigGenerator(),
metrics.NewCtrlFuncMetrics(),
)
require.NoError(t, err)
return kongClient
Expand Down Expand Up @@ -1002,6 +1003,7 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) {
configBuilder,
originalCache,
fallbackConfigGenerator,
metrics.NewCtrlFuncMetrics(),
)
require.NoError(t, err)

Expand Down Expand Up @@ -1114,6 +1116,7 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) {
configBuilder,
originalCache,
fallbackConfigGenerator,
metrics.NewCtrlFuncMetrics(),
)
require.NoError(t, err)

Expand Down
5 changes: 4 additions & 1 deletion internal/manager/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
"net/http"
"net/url"
"os"
Expand Down Expand Up @@ -185,7 +186,8 @@ func Run(
}

setupLog.Info("Starting Admission Server")
if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer); err != nil {
promMetrics := metrics.NewCtrlFuncMetrics()
if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer, promMetrics); err != nil {
return err
}

Expand All @@ -207,6 +209,7 @@ func Run(
configTranslator,
cache,
fallbackConfigGenerator,
promMetrics,
)
if err != nil {
return fmt.Errorf("failed to initialize kong data-plane client: %w", err)
Expand Down
3 changes: 3 additions & 0 deletions internal/manager/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
"io"
"time"

Expand Down Expand Up @@ -195,6 +196,7 @@ func setupAdmissionServer(
logger logr.Logger,
translatorFeatures translator.FeatureFlags,
storer store.Storer,
promMetrics *metrics.CtrlFuncMetrics,
) error {
admissionLogger := logger.WithName("admission-server")

Expand All @@ -214,6 +216,7 @@ func setupAdmissionServer(
storer,
),
ReferenceIndexers: referenceIndexers,
PromMetrics: promMetrics,
Logger: admissionLogger,
}, admissionLogger)
if err != nil {
Expand Down
37 changes: 37 additions & 0 deletions internal/metrics/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"net"
"strconv"
"sync"
"time"

Expand All @@ -27,6 +28,8 @@ type CtrlFuncMetrics struct {
ConfigPushDuration *prometheus.HistogramVec

ConfigPushSuccessTime *prometheus.GaugeVec

AdmissionCount *prometheus.CounterVec
}

const (
Expand Down Expand Up @@ -70,13 +73,24 @@ const (
DataplaneKey string = "dataplane"
)

const (
// AllowedKey defines the key of the metric label indicating admission was allowed.
AllowedKey string = "allowed"
)

const (
// AdmissionResourceKey defines the name of the metric label indicating which dataplane this time series is relevant for.
AdmissionResourceKey string = "resource"
)

const (
MetricNameConfigPushCount = "ingress_controller_configuration_push_count"
MetricNameConfigPushBrokenResources = "ingress_controller_configuration_push_broken_resource_count"
MetricNameConfigPushSuccessTime = "ingress_controller_configuration_push_last_successful"
MetricNameTranslationCount = "ingress_controller_translation_count"
MetricNameTranslationBrokenResources = "ingress_controller_translation_broken_resource_count"
MetricNameConfigPushDuration = "ingress_controller_configuration_push_duration_milliseconds"
MetricNameAdmissionCount = "ingress_controller_admission_count"
)

var _lock sync.Mutex
Expand Down Expand Up @@ -168,12 +182,27 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics {
[]string{DataplaneKey},
)

controllerMetrics.AdmissionCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: MetricNameAdmissionCount,
Help: fmt.Sprintf(
"Count of admissions processed by Kong. "+
"`%s` describes wheter an admission was allowed. "+
"`%s` describes the resouce under admission. ",
AllowedKey,
AdmissionResourceKey,
),
},
[]string{AllowedKey, AdmissionResourceKey},
)

metrics.Registry.Unregister(controllerMetrics.ConfigPushCount)
metrics.Registry.Unregister(controllerMetrics.ConfigPushBrokenResources)
metrics.Registry.Unregister(controllerMetrics.TranslationCount)
metrics.Registry.Unregister(controllerMetrics.TranslationBrokenResources)
metrics.Registry.Unregister(controllerMetrics.ConfigPushDuration)
metrics.Registry.Unregister(controllerMetrics.ConfigPushSuccessTime)
metrics.Registry.Unregister(controllerMetrics.AdmissionCount)

metrics.Registry.MustRegister(
controllerMetrics.ConfigPushCount,
Expand All @@ -182,6 +211,7 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics {
controllerMetrics.TranslationBrokenResources,
controllerMetrics.ConfigPushDuration,
controllerMetrics.ConfigPushSuccessTime,
controllerMetrics.AdmissionCount,
)

return controllerMetrics
Expand Down Expand Up @@ -223,6 +253,13 @@ func (c *CtrlFuncMetrics) RecordTranslationBrokenResources(count int) {
c.TranslationBrokenResources.Set(float64(count))
}

func (c *CtrlFuncMetrics) RecordAdmissionCount(allowed bool, resource string) {
c.ConfigPushCount.With(prometheus.Labels{
AllowedKey: strconv.FormatBool(allowed),
AdmissionResourceKey: resource,
}).Inc()
}

type recordOption func(prometheus.Labels) prometheus.Labels

func withError(err error) recordOption {
Expand Down
1 change: 1 addition & 0 deletions test/envtest/metrics_envtest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func TestMetricsAreServed(t *testing.T) {
metrics.MetricNameTranslationBrokenResources,
metrics.MetricNameConfigPushDuration,
metrics.MetricNameConfigPushSuccessTime,
metrics.MetricNameAdmissionCount,
}

metricsURL := fmt.Sprintf("http://%s/metrics", cfg.MetricsAddr)
Expand Down

0 comments on commit fb7a9eb

Please sign in to comment.