Replace deprecated gcr.io/kubebuilder/kube-rbac-proxy image

- Add the `WithAuthenticationAndAuthorization` filter to support deprecation of kube-rbac-proxy image from the kubebuilder repository - Add metrics-reader cluster role binding and the metrics service - Add e2e test case to validate the `/metrics` endpoint Signed-off-by: Sai Ramesh Vanka <[email protected]>
openshift · Dec 30, 2024 · f7cbf35 · f7cbf35
1 parent a6d4fc8
commit f7cbf35
Show file tree

Hide file tree

Showing 20 changed files with 262 additions and 65 deletions.
diff --git a/bundle-ocp/manifests/instaslice-operator-controller-manager-metrics-service_v1_service.yaml b/bundle-ocp/manifests/instaslice-operator-controller-manager-metrics-service_v1_service.yaml
@@ -3,7 +3,6 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/instance: controller-manager-metrics-service
     app.kubernetes.io/managed-by: kustomize

diff --git a/...anifests/instaslice-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml b/...anifests/instaslice-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml
@@ -3,7 +3,6 @@ kind: ClusterRole
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/instance: metrics-reader
     app.kubernetes.io/managed-by: kustomize

diff --git a/bundle-ocp/manifests/instaslice-operator.clusterserviceversion.yaml b/bundle-ocp/manifests/instaslice-operator.clusterserviceversion.yaml
@@ -248,29 +248,6 @@ spec:
                 - mountPath: /tmp/k8s-webhook-server/serving-certs
                   name: cert
                   readOnly: true
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: registry.redhat.io/openshift4/ose-kube-rbac-proxy-rhel9@sha256:90b19de8a962e4b99cf336af1a51e6288ce493e35644f3fb8b9077b76e7ff98a
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  limits:
-                    cpu: 500m
-                    memory: 128Mi
-                  requests:
-                    cpu: 5m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
               securityContext:
                 runAsNonRoot: true
                 seccompProfile:

diff --git a/bundle/manifests/instaslice-operator-controller-manager-metrics-service_v1_service.yaml b/bundle/manifests/instaslice-operator-controller-manager-metrics-service_v1_service.yaml
@@ -3,7 +3,6 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/instance: controller-manager-metrics-service
     app.kubernetes.io/managed-by: kustomize

diff --git a/...anifests/instaslice-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml b/...anifests/instaslice-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml
@@ -3,7 +3,6 @@ kind: ClusterRole
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/instance: metrics-reader
     app.kubernetes.io/managed-by: kustomize

diff --git a/bundle/manifests/instaslice-operator.clusterserviceversion.yaml b/bundle/manifests/instaslice-operator.clusterserviceversion.yaml
@@ -243,29 +243,6 @@ spec:
                 - mountPath: /tmp/k8s-webhook-server/serving-certs
                   name: cert
                   readOnly: true
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  limits:
-                    cpu: 500m
-                    memory: 128Mi
-                  requests:
-                    cpu: 5m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
               securityContext:
                 runAsNonRoot: false
               serviceAccountName: instaslice-operator-controller-manager

diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -33,6 +33,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsfilters "sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -66,7 +67,7 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", false,
+	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set the metrics endpoint is served securely")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
@@ -104,6 +105,9 @@ func main() {
 			BindAddress:   metricsAddr,
 			SecureServing: secureMetrics,
 			TLSOpts:       tlsOpts,
+			// enable the authN and authZ filter to support the deprecation of gcr.io/kubebuilder/kube-rbac-proxy image
+			// Ref: https://github.com/openshift/instaslice-operator/issues/291
+			FilterProvider: metricsfilters.WithAuthenticationAndAuthorization,
 		},
 		WebhookServer:          webhookServer,
 		HealthProbeBindAddress: probeAddr,

diff --git a/cmd/daemonset/main.go b/cmd/daemonset/main.go
@@ -33,6 +33,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsfilters "sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
@@ -65,7 +66,7 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", false,
+	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set the metrics endpoint is served securely")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
@@ -106,6 +107,9 @@ func main() {
 			BindAddress:   metricsAddr,
 			SecureServing: secureMetrics,
 			TLSOpts:       tlsOpts,
+			// enable the authN and authZ filter to support the deprecation of gcr.io/kubebuilder/kube-rbac-proxy image
+			// Ref: https://github.com/openshift/instaslice-operator/issues/291
+			FilterProvider: metricsfilters.WithAuthenticationAndAuthorization,
 		},
 		WebhookServer:          webhookServer,
 		HealthProbeBindAddress: probeAddr,

diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -25,12 +25,14 @@ resources:
 - ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
+- manager_metrics_service.yaml
 
 patches:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-- path: manager_auth_proxy_patch.yaml
+#- path: manager_auth_proxy_patch.yaml
+- path: manager_config_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml
@@ -8,3 +8,7 @@ spec:
     spec:
       containers:
       - name: manager
+        args:
+        - "--health-probe-bind-address=:8081"
+        - "--metrics-bind-address=:8443"
+        - "--leader-elect"
diff --git a/config/default/manager_metrics_service.yaml b/config/default/manager_metrics_service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: controller-manager
+    app.kubernetes.io/name: service
+    app.kubernetes.io/instance: controller-manager-metrics-service
+    app.kubernetes.io/created-by: instaslice-operator
+    app.kubernetes.io/part-of: instaslice-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: controller-manager-metrics-service
+  namespace: system
+spec:
+  ports:
+  - name: https
+    port: 8443
+    protocol: TCP
+    targetPort: 8443
+  selector:
+    control-plane: controller-manager
diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: registry.redhat.io/dynamic-accelerator-slicer-tech-preview/instaslice-operator-rhel9
-  newTag: 0.0.1
+  newName: quay.io/amalvank/instaslicev2-controller
+  newTag: test-e2e
diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml
@@ -4,7 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/name: clusterrole
     app.kubernetes.io/instance: metrics-reader
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/part-of: instaslice-operator
     app.kubernetes.io/managed-by: kustomize

diff --git a/config/rbac/auth_proxy_metrics_reader_role_binding.yaml b/config/rbac/auth_proxy_metrics_reader_role_binding.yaml
@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: clusterrolebinding
+    app.kubernetes.io/instance: metrics-reader-rolebinding
+    app.kubernetes.io/created-by: instaslice-operator
+    app.kubernetes.io/part-of: instaslice-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system
diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml
@@ -4,7 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/name: clusterrole
     app.kubernetes.io/instance: proxy-role
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/part-of: instaslice-operator
     app.kubernetes.io/managed-by: kustomize

diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml
@@ -4,7 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/name: clusterrolebinding
     app.kubernetes.io/instance: proxy-rolebinding
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/part-of: instaslice-operator
     app.kubernetes.io/managed-by: kustomize

diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml
@@ -5,7 +5,6 @@ metadata:
     control-plane: controller-manager
     app.kubernetes.io/name: service
     app.kubernetes.io/instance: controller-manager-metrics-service
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: instaslice-operator
     app.kubernetes.io/part-of: instaslice-operator
     app.kubernetes.io/managed-by: kustomize

diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
@@ -12,10 +12,11 @@ resources:
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.
-- auth_proxy_service.yaml
+#- auth_proxy_service.yaml
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
 - auth_proxy_client_clusterrole.yaml
+- auth_proxy_metrics_reader_role_binding.yaml
 #- instaslice-operator-scc.yaml
 #- openshift_cluster_role.yaml
 #- openshift_scc_cluster_role_binding.yaml