Skip to content

Commit

Permalink
Add validating and mutating webhook config for supervisor mode
Browse files Browse the repository at this point in the history
- Refactor supervisor config folder and generate webhook config and manifest
- Enable update validating for vm-operator VSphereMachine to block in-place update of following fields.
	- ImageName
	- ClassName
	- StorageClass
	- MinHardwareVersion

Signed-off-by: Gong Zhang <[email protected]>
  • Loading branch information
zhanggbj committed May 14, 2024
1 parent e2d3d1d commit fce5c50
Show file tree
Hide file tree
Showing 26 changed files with 345 additions and 28 deletions.
10 changes: 8 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -255,9 +255,10 @@ LDFLAGS ?= $(shell hack/version.sh)
# Allow overriding manifest generation destination directory
MANIFEST_ROOT ?= ./config
CRD_ROOT ?= $(MANIFEST_ROOT)/default/crd/bases
SUPERVISOR_CRD_ROOT ?= $(MANIFEST_ROOT)/supervisor/crd
SUPERVISOR_CRD_ROOT ?= $(MANIFEST_ROOT)/supervisor/crd/bases
VCSIM_CRD_ROOT ?= $(VCSIM_DIR)/config/crd/bases
WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
SUPERVISOR_WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/supervisor/webhook
RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
VCSIM_RBAC_ROOT ?= $(VCSIM_DIR)/config/rbac
NETOP_RBAC_ROOT ?= $(NETOP_DIR)/config/rbac
Expand All @@ -281,7 +282,7 @@ generate: ## Run all generate targets

.PHONY: generate-manifests
generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
$(MAKE) clean-generated-yaml SRC_DIRS="$(CRD_ROOT),$(SUPERVISOR_CRD_ROOT),./config/webhook/manifests.yaml"
$(MAKE) clean-generated-yaml SRC_DIRS="$(CRD_ROOT),$(SUPERVISOR_CRD_ROOT),./config/webhook/manifests.yaml,./config/supervisor/webhook/manifests.yaml"
$(CONTROLLER_GEN) \
paths=./apis/v1alpha3 \
paths=./apis/v1alpha4 \
Expand All @@ -291,6 +292,11 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
output:crd:dir=$(CRD_ROOT) \
output:webhook:dir=$(WEBHOOK_ROOT) \
webhook
# Generate webhook manifests for supervisor mode separately.
$(CONTROLLER_GEN) \
paths=./internal/webhooks/vmware\
output:webhook:dir=$(SUPERVISOR_WEBHOOK_ROOT) \
webhook
$(CONTROLLER_GEN) \
paths=./ \
paths=./controllers/... \
Expand Down
1 change: 0 additions & 1 deletion config/base/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ patchesStrategicMerge:
- manager_pull_policy.yaml
- manager_credentials_patch.yaml
- manager_webhook_patch.yaml
- webhookcainjection_patch.yaml
- manager_role_aggregation_patch.yaml
# Protect the /metrics endpoint by putting it behind auth.
# Only one of manager_auth_proxy_patch.yaml and
Expand Down
12 changes: 7 additions & 5 deletions config/supervisor/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ commonLabels:
cluster.x-k8s.io/v1beta1: v1beta1
resources:
- ../base
- crd/vmware.infrastructure.cluster.x-k8s.io_vspheremachines.yaml
- crd/vmware.infrastructure.cluster.x-k8s.io_vsphereclusters.yaml
- crd/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml
- crd/vmware.infrastructure.cluster.x-k8s.io_vsphereclustertemplates.yaml
- crd/vmware.infrastructure.cluster.x-k8s.io_providerserviceaccounts.yaml
- crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachines.yaml
- crd/bases/vmware.infrastructure.cluster.x-k8s.io_vsphereclusters.yaml
- crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml
- crd/bases/vmware.infrastructure.cluster.x-k8s.io_vsphereclustertemplates.yaml
- crd/bases/vmware.infrastructure.cluster.x-k8s.io_providerserviceaccounts.yaml
- ./webhook

patchesStrategicMerge:
- add-configmap-env-vars.yaml

10 changes: 10 additions & 0 deletions config/supervisor/webhook/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
resources:
- manifests.yaml

namePrefix: supervisor-

configurations:
- kustomizeconfig.yaml

patchesStrategicMerge:
- webhookcainjection_patch.yaml
20 changes: 20 additions & 0 deletions config/supervisor/webhook/kustomizeconfig.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
nameReference:
- kind: Service
version: v1
fieldSpecs:
- kind: ValidatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/name
- kind: MutatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/name

namespace:
- kind: ValidatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/namespace
create: true
- kind: MutatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/namespace
create: true
54 changes: 54 additions & 0 deletions config/supervisor/webhook/manifests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: mutating-webhook-configuration
webhooks:
- admissionReviewVersions:
- v1beta1
clientConfig:
service:
name: webhook-service
namespace: system
path: /mutate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine
failurePolicy: Fail
matchPolicy: Equivalent
name: default.vspheremachine.vmware.infrastructure.cluster.x-k8s.io
rules:
- apiGroups:
- vmware.infrastructure.cluster.x-k8s.io
apiVersions:
- v1beta1
operations:
- CREATE
- UPDATE
resources:
- vspheremachines
sideEffects: None
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
name: validating-webhook-configuration
webhooks:
- admissionReviewVersions:
- v1beta1
clientConfig:
service:
name: webhook-service
namespace: system
path: /validate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine
failurePolicy: Fail
matchPolicy: Equivalent
name: validation.vspheremachine.vmware.cluster.infrastructure.cluster.x-k8s.io
rules:
- apiGroups:
- vmware.infrastructure.cluster.x-k8s.io
apiVersions:
- v1beta1
operations:
- CREATE
- UPDATE
resources:
- vspheremachines
sideEffects: None
File renamed without changes.
3 changes: 3 additions & 0 deletions config/webhook/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@ resources:

configurations:
- kustomizeconfig.yaml

patchesStrategicMerge:
- webhookcainjection_patch.yaml
10 changes: 5 additions & 5 deletions config/webhook/manifests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ webhooks:
path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-vspherevm
failurePolicy: Fail
matchPolicy: Equivalent
name: default.vspherevm.infrastructure.x-k8s.io
name: default.vspherevm.infrastructure.cluster.x-k8s.io
rules:
- apiGroups:
- infrastructure.cluster.x-k8s.io
Expand All @@ -103,7 +103,7 @@ webhooks:
path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vsphereclustertemplate
failurePolicy: Fail
matchPolicy: Equivalent
name: validation.vsphereclustertemplate.infrastructure.x-k8s.io
name: validation.vsphereclustertemplate.infrastructure.cluster.x-k8s.io
rules:
- apiGroups:
- infrastructure.cluster.x-k8s.io
Expand Down Expand Up @@ -145,7 +145,7 @@ webhooks:
path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine
failurePolicy: Fail
matchPolicy: Equivalent
name: validation.vspheremachine.infrastructure.x-k8s.io
name: validation.vspheremachine.infrastructure.cluster.x-k8s.io
rules:
- apiGroups:
- infrastructure.cluster.x-k8s.io
Expand All @@ -166,7 +166,7 @@ webhooks:
path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachinetemplate
failurePolicy: Fail
matchPolicy: Equivalent
name: validation.vspheremachinetemplate.infrastructure.x-k8s.io
name: validation.vspheremachinetemplate.infrastructure.cluster.x-k8s.io
rules:
- apiGroups:
- infrastructure.cluster.x-k8s.io
Expand All @@ -187,7 +187,7 @@ webhooks:
path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-vspherevm
failurePolicy: Fail
matchPolicy: Equivalent
name: validation.vspherevm.infrastructure.x-k8s.io
name: validation.vspherevm.infrastructure.cluster.x-k8s.io
rules:
- apiGroups:
- infrastructure.cluster.x-k8s.io
Expand Down
15 changes: 15 additions & 0 deletions config/webhook/webhookcainjection_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# This patch add annotation to admission webhook config and
# the variables $(NAMESPACE) and $(CERTIFICATENAME) will be substituted by kustomize.
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
name: validating-webhook-configuration
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: mutating-webhook-configuration
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
2 changes: 1 addition & 1 deletion controllers/vmware/test/controllers_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func getTestEnv() (*envtest.Environment, *rest.Config) {

localTestEnv := &envtest.Environment{
CRDDirectoryPaths: []string{
filepath.Join(root, "config", "supervisor", "crd"),
filepath.Join(root, "config", "supervisor", "crd", "bases"),
filepath.Join(root, "config", "deployments", "integration-tests", "crds"),
filepath.Join(clusterAPIDir, "config", "crd", "bases"),
},
Expand Down
6 changes: 5 additions & 1 deletion internal/test/helpers/envtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import (
infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
"sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vcsim"
"sigs.k8s.io/cluster-api-provider-vsphere/internal/webhooks"
vmwarewebhooks "sigs.k8s.io/cluster-api-provider-vsphere/internal/webhooks/vmware"
capvcontext "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context"
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/manager"
)
Expand Down Expand Up @@ -84,7 +85,7 @@ func init() {

crdPaths := []string{
filepath.Join(root, "config", "default", "crd", "bases"),
filepath.Join(root, "config", "supervisor", "crd"),
filepath.Join(root, "config", "supervisor", "crd", "bases"),
}

// append CAPI CRDs path
Expand Down Expand Up @@ -159,6 +160,9 @@ func NewTestEnvironment(ctx context.Context) *TestEnvironment {
Password: simr.Password(),
}
managerOpts.AddToManager = func(_ context.Context, _ *capvcontext.ControllerManagerContext, mgr ctrlmgr.Manager) error {
if err := (&vmwarewebhooks.VSphereMachineWebhook{}).SetupWebhookWithManager(mgr); err != nil {
return err
}
if err := (&webhooks.VSphereClusterTemplateWebhook{}).SetupWebhookWithManager(mgr); err != nil {
return err
}
Expand Down
3 changes: 2 additions & 1 deletion internal/webhooks/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import (
"k8s.io/apimachinery/pkg/util/validation/field"
)

func aggregateObjErrors(gk schema.GroupKind, name string, allErrs field.ErrorList) error {
// AggregateObjErrors aggregates a list of field errors into a single Invalid API error.
func AggregateObjErrors(gk schema.GroupKind, name string, allErrs field.ErrorList) error {
if len(allErrs) == 0 {
return nil
}
Expand Down
105 changes: 105 additions & 0 deletions internal/webhooks/vmware/vspheremachine.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package vmware is the package for webhooks of vmware resources.
package vmware

import (
"context"
"fmt"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation/field"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
"sigs.k8s.io/cluster-api-provider-vsphere/internal/webhooks"
)

// +kubebuilder:webhook:verbs=create;update,path=/validate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=vmware.infrastructure.cluster.x-k8s.io,resources=vspheremachines,versions=v1beta1,name=validation.vspheremachine.vmware.cluster.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1
// +kubebuilder:webhook:verbs=create;update,path=/mutate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspheremachine,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=vmware.infrastructure.cluster.x-k8s.io,resources=vspheremachines,versions=v1beta1,name=default.vspheremachine.vmware.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1

// VSphereMachineWebhook implements a validation and defaulting webhook for VSphereMachine.
type VSphereMachineWebhook struct{}

var _ webhook.CustomValidator = &VSphereMachineWebhook{}
var _ webhook.CustomDefaulter = &VSphereMachineWebhook{}

func (webhook *VSphereMachineWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
For(&vmwarev1.VSphereMachine{}).
WithValidator(webhook).
WithDefaulter(webhook).
Complete()
}

// Default implements webhook.Defaulter so a webhook will be registered for the type.
func (webhook *VSphereMachineWebhook) Default(_ context.Context, _ runtime.Object) error {
return nil
}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type.
func (webhook *VSphereMachineWebhook) ValidateCreate(_ context.Context, _ runtime.Object) (admission.Warnings, error) {
return nil, nil
}

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
func (webhook *VSphereMachineWebhook) ValidateUpdate(_ context.Context, oldRaw runtime.Object, newRaw runtime.Object) (admission.Warnings, error) {
var allErrs field.ErrorList

newTyped, ok := newRaw.(*vmwarev1.VSphereMachine)
if !ok {
return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a VSphereMachine but got a %T", newRaw))
}

oldTyped, ok := oldRaw.(*vmwarev1.VSphereMachine)
if !ok {
return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a VSphereMachine but got a %T", oldRaw))
}

newSpec, oldSpec := newTyped.Spec, oldTyped.Spec

// In VM operator, following fields are immutable, so CAPV should not allow to update them.
// - ImageName
// - ClassName
// - StorageClass
// - MinHardwareVersion
if newSpec.ImageName != oldSpec.ImageName {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "imageName"), "cannot be modified"))
}

if newSpec.ClassName != oldSpec.ClassName {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "className"), "cannot be modified"))
}

if newSpec.StorageClass != oldSpec.StorageClass {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "storageClass"), "cannot be modified"))
}

if newSpec.MinHardwareVersion != oldSpec.MinHardwareVersion {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "minHardwareVersion"), "cannot be modified"))
}

return nil, webhooks.AggregateObjErrors(newTyped.GroupVersionKind().GroupKind(), newTyped.Name, allErrs)
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
func (webhook *VSphereMachineWebhook) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) {
return nil, nil
}
Loading

0 comments on commit fce5c50

Please sign in to comment.