Merge pull request #2651 from zhanggbj/vmop_vm_webhook

✨ Add validating and mutating webhook for supervisor mode

Makefile

@@ -256,9 +256,10 @@ LDFLAGS ?= $(shell hack/version.sh)
 # Allow overriding manifest generation destination directory
 MANIFEST_ROOT ?= ./config
 CRD_ROOT ?= $(MANIFEST_ROOT)/default/crd/bases
-SUPERVISOR_CRD_ROOT ?= $(MANIFEST_ROOT)/supervisor/crd
+SUPERVISOR_CRD_ROOT ?= $(MANIFEST_ROOT)/supervisor/crd/bases
 VCSIM_CRD_ROOT ?= $(VCSIM_DIR)/config/crd/bases
-WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
+WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/govmomi/webhook
+SUPERVISOR_WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/supervisor/webhook
 RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 VCSIM_RBAC_ROOT ?= $(VCSIM_DIR)/config/rbac
 NETOP_RBAC_ROOT ?= $(NETOP_DIR)/config/rbac
@@ -282,7 +283,7 @@ generate: ## Run all generate targets
 
 .PHONY: generate-manifests
 generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
-	$(MAKE) clean-generated-yaml SRC_DIRS="$(CRD_ROOT),$(SUPERVISOR_CRD_ROOT),./config/webhook/manifests.yaml"
+	$(MAKE) clean-generated-yaml SRC_DIRS="$(CRD_ROOT),$(SUPERVISOR_CRD_ROOT),./config/govmomi/webhook/manifests.yaml,./config/supervisor/webhook/manifests.yaml"
 	$(CONTROLLER_GEN) \
 		paths=./apis/v1alpha3 \
 		paths=./apis/v1alpha4 \
@@ -292,6 +293,11 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
 		output:crd:dir=$(CRD_ROOT) \
 		output:webhook:dir=$(WEBHOOK_ROOT) \
 		webhook
+	# Generate webhook manifests for supervisor mode separately.
+	$(CONTROLLER_GEN) \
+		paths=./internal/webhooks/vmware\
+		output:webhook:dir=$(SUPERVISOR_WEBHOOK_ROOT) \
+		webhook
 	$(CONTROLLER_GEN) \
 		paths=./ \
 		paths=./controllers/... \

config/base/kustomization.yaml

@@ -12,15 +12,13 @@ resources:
 bases:
   - ../rbac
   - ../manager
-  - ../webhook
   - ../certmanager
 
 patchesStrategicMerge:
   - manager_image_patch.yaml
   - manager_pull_policy.yaml
   - manager_credentials_patch.yaml
   - manager_webhook_patch.yaml
-  - webhookcainjection_patch.yaml
   - manager_role_aggregation_patch.yaml
     # Protect the /metrics endpoint by putting it behind auth.
     # Only one of manager_auth_proxy_patch.yaml and
@@ -40,30 +38,4 @@ patchesStrategicMerge:
 # Uncomment 'CAINJECTION' in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
 #- webhookcainjection_patch.yaml
-vars:
-  - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-    objref:
-      kind: Certificate
-      group: cert-manager.io
-      version: v1
-      name: serving-cert # this name should match the one in certificate.yaml
-    fieldref:
-      fieldpath: metadata.namespace
-  - name: CERTIFICATE_NAME
-    objref:
-      kind: Certificate
-      group: cert-manager.io
-      version: v1
-      name: serving-cert # this name should match the one in certificate.yaml
-  - name: SERVICE_NAMESPACE # namespace of the service
-    objref:
-      kind: Service
-      version: v1
-      name: webhook-service
-    fieldref:
-      fieldpath: metadata.namespace
-  - name: SERVICE_NAME
-    objref:
-      kind: Service
-      version: v1
-      name: webhook-service
+

config/default/kustomization.yaml

@@ -12,3 +12,32 @@ commonLabels:
 resources:
   - ../base
   - ./crd
+  - ../govmomi/webhook
+
+vars:
+  - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: serving-cert # this name should match the one in certificate.yaml
+    fieldref:
+      fieldpath: metadata.namespace
+  - name: CERTIFICATE_NAME
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: serving-cert # this name should match the one in certificate.yaml
+  - name: SERVICE_NAMESPACE # namespace of the service
+    objref:
+      kind: Service
+      version: v1
+      name: webhook-service
+    fieldref:
+      fieldpath: metadata.namespace
+  - name: SERVICE_NAME
+    objref:
+      kind: Service
+      version: v1
+      name: webhook-service

config/webhook/kustomization.yaml → config/govmomi/webhook/kustomization.yaml

@@ -4,3 +4,6 @@ resources:
 
 configurations:
   - kustomizeconfig.yaml
+
+patchesStrategicMerge:
+  - webhookcainjection_patch.yaml

config/webhook/kustomizeconfig.yaml → config/govmomi/webhook/kustomizeconfig.yaml

@@ -9,7 +9,6 @@ nameReference:
     group: admissionregistration.k8s.io
     path: webhooks/clientConfig/service/name
 
-
 namespace:
 - kind: ValidatingWebhookConfiguration
   group: admissionregistration.k8s.io

config/webhook/manifests.yaml → config/govmomi/webhook/manifests.yaml

@@ -76,7 +76,7 @@ webhooks:
       path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-vspherevm
   failurePolicy: Fail
   matchPolicy: Equivalent
-  name: default.vspherevm.infrastructure.x-k8s.io
+  name: default.vspherevm.infrastructure.cluster.x-k8s.io
   rules:
   - apiGroups:
     - infrastructure.cluster.x-k8s.io
@@ -103,7 +103,7 @@ webhooks:
       path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vsphereclustertemplate
   failurePolicy: Fail
   matchPolicy: Equivalent
-  name: validation.vsphereclustertemplate.infrastructure.x-k8s.io
+  name: validation.vsphereclustertemplate.infrastructure.cluster.x-k8s.io
   rules:
   - apiGroups:
     - infrastructure.cluster.x-k8s.io
@@ -145,7 +145,7 @@ webhooks:
       path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine
   failurePolicy: Fail
   matchPolicy: Equivalent
-  name: validation.vspheremachine.infrastructure.x-k8s.io
+  name: validation.vspheremachine.infrastructure.cluster.x-k8s.io
   rules:
   - apiGroups:
     - infrastructure.cluster.x-k8s.io
@@ -166,7 +166,7 @@ webhooks:
       path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachinetemplate
   failurePolicy: Fail
   matchPolicy: Equivalent
-  name: validation.vspheremachinetemplate.infrastructure.x-k8s.io
+  name: validation.vspheremachinetemplate.infrastructure.cluster.x-k8s.io
   rules:
   - apiGroups:
     - infrastructure.cluster.x-k8s.io
@@ -187,7 +187,7 @@ webhooks:
       path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vspherevm
   failurePolicy: Fail
   matchPolicy: Equivalent
-  name: validation.vspherevm.infrastructure.x-k8s.io
+  name: validation.vspherevm.infrastructure.cluster.x-k8s.io
   rules:
   - apiGroups:
     - infrastructure.cluster.x-k8s.io

config/supervisor/kustomization.yaml

@@ -10,11 +10,40 @@ commonLabels:
   cluster.x-k8s.io/v1beta1: v1beta1
 resources:
   - ../base
-  - crd/vmware.infrastructure.cluster.x-k8s.io_vspheremachines.yaml
-  - crd/vmware.infrastructure.cluster.x-k8s.io_vsphereclusters.yaml
-  - crd/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml
-  - crd/vmware.infrastructure.cluster.x-k8s.io_vsphereclustertemplates.yaml
-  - crd/vmware.infrastructure.cluster.x-k8s.io_providerserviceaccounts.yaml
+  - crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachines.yaml
+  - crd/bases/vmware.infrastructure.cluster.x-k8s.io_vsphereclusters.yaml
+  - crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml
+  - crd/bases/vmware.infrastructure.cluster.x-k8s.io_vsphereclustertemplates.yaml
+  - crd/bases/vmware.infrastructure.cluster.x-k8s.io_providerserviceaccounts.yaml
+  - ./webhook
 
 patchesStrategicMerge:
   - add-configmap-env-vars.yaml
+
+vars:
+  - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: serving-cert # this name should match the one in certificate.yaml
+    fieldref:
+      fieldpath: metadata.namespace
+  - name: CERTIFICATE_NAME
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: serving-cert # this name should match the one in certificate.yaml
+  - name: SERVICE_NAMESPACE # namespace of the service
+    objref:
+      kind: Service
+      version: v1
+      name: webhook-service
+    fieldref:
+      fieldpath: metadata.namespace
+  - name: SERVICE_NAME
+    objref:
+      kind: Service
+      version: v1
+      name: webhook-service

config/supervisor/webhook/kustomization.yaml

@@ -0,0 +1,9 @@
+resources:
+- service.yaml
+- manifests.yaml
+
+configurations:
+  - kustomizeconfig.yaml
+
+patchesStrategicMerge:
+  - webhookcainjection_patch.yaml

config/supervisor/webhook/kustomizeconfig.yaml

@@ -0,0 +1,20 @@
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: ValidatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+  - kind: MutatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+
+namespace:
+- kind: ValidatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+- kind: MutatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true

config/supervisor/webhook/manifests.yaml

@@ -0,0 +1,54 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: default.vspheremachine.vmware.infrastructure.cluster.x-k8s.io
+  rules:
+  - apiGroups:
+    - vmware.infrastructure.cluster.x-k8s.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - vspheremachines
+  sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: validation.vspheremachine.vmware.infrastructure.cluster.x-k8s.io
+  rules:
+  - apiGroups:
+    - vmware.infrastructure.cluster.x-k8s.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - vspheremachines
+  sideEffects: None

config/supervisor/webhook/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: webhook-service
+  namespace: system
+spec:
+  ports:
+    - port: 443
+      targetPort: webhook-server

config/supervisor/webhook/webhookcainjection_patch.yaml

@@ -0,0 +1,15 @@
+# This patch add annotation to admission webhook config and
+# the variables $(NAMESPACE) and $(CERTIFICATENAME) will be substituted by kustomize.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

controllers/vmware/test/controllers_suite_test.go

@@ -40,6 +40,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers"
 )
 
 var (
@@ -77,13 +78,16 @@ func getTestEnv() (*envtest.Environment, *rest.Config) {
 
 	localTestEnv := &envtest.Environment{
 		CRDDirectoryPaths: []string{
-			filepath.Join(root, "config", "supervisor", "crd"),
+			filepath.Join(root, "config", "supervisor", "crd", "bases"),
 			filepath.Join(root, "config", "deployments", "integration-tests", "crds"),
 			filepath.Join(clusterAPIDir, "config", "crd", "bases"),
 		},
 		ControlPlaneStopTimeout: 60 * time.Second,
 	}
 
+	configPath := filepath.Clean(filepath.Join(root, "config", "supervisor", "webhook", "manifests.yaml"))
+	helpers.InitializeWebhookInEnvironment(localTestEnv, configPath)
+
 	localCfg, err := localTestEnv.Start()
 	Expect(err).ToNot(HaveOccurred())
 	Expect(localCfg).ToNot(BeNil())