✨ Enable NSX LB healthcheck on kube-apiserver (kubernetes-sigs#3128)

* Enable NSX LB healthcheck on kube-apiserver

In the VPC network, the vm-operator Pod and Kubernetes
Clusters are in different VPCs. TCP readiness probe does
not work across VPC. Use LB 's healthcheck instead. This
annotation is agnostic to LB providers. NSX LB supports
it to enable healthcheck while AVI LB has healthcheck
enabled by default.

* Update nsx-operator API mod

There is a group refactor in nsx-operator API, so we need to
update to latest version.

* Fixup code review comments

.golangci.yml

@@ -122,8 +122,11 @@ linters-settings:
         alias: vmwarev1
       # VMware Operator
       - pkg: "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
-        alias: vmoprv1 
-      # CABPK
+        alias: vmoprv1
+      # NSX Operator
+      - pkg: "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
+        alias: nsxvpcv1
+    # CABPK
       - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1
         alias: bootstrapv1
       # KCP

config/rbac/role.yaml

@@ -187,6 +187,19 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - crd.nsx.vmware.com
+  resources:
+  - subnetsets
+  - subnetsets/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - infrastructure.cluster.x-k8s.io
   resources:
@@ -334,19 +347,6 @@ rules:
   - get
   - list
   - watch
-- apiGroups:
-  - nsx.vmware.com
-  resources:
-  - subnetsets
-  - subnetsets/status
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:

controllers/vmware/vspherecluster_reconciler.go

@@ -62,7 +62,7 @@ type ClusterReconciler struct {
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclustertemplates,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=crd.nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.com,resources=virtualnetworks;virtualnetworks/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies;virtualmachinesetresourcepolicies/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices;virtualmachineservices/status,verbs=get;list;watch;create;update;patch;delete

go.mod

@@ -8,7 +8,7 @@ replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.c
 
 require (
 	github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d
-	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0
+	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea
 	// The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-tests
 	github.com/vmware-tanzu/vm-operator/api v1.8.6
 	github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505

go.sum

@@ -270,8 +270,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:cgx9UH/r53bKU/Gbv8IPsUZ34bj5+ItijA2JCUS3kVk=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea h1:9BE++G23adnANH0J4UCqysUHoryCy678Yra8SCZ39FQ=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
 github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU=
 github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU=

pkg/manager/manager.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	"gopkg.in/fsnotify.v1"
@@ -65,7 +65,7 @@ func New(ctx context.Context, opts Options) (Manager, error) {
 	_ = vmoprv1.AddToScheme(opts.Scheme)
 	_ = ncpv1.AddToScheme(opts.Scheme)
 	_ = netopv1.AddToScheme(opts.Scheme)
-	_ = nsxopv1.AddToScheme(opts.Scheme)
+	_ = nsxvpcv1.AddToScheme(opts.Scheme)
 	_ = topologyv1.AddToScheme(opts.Scheme)
 	_ = ipamv1.AddToScheme(opts.Scheme)
 

pkg/services/network/constants.go

@@ -19,7 +19,7 @@ package network
 
 import (
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 )
 
@@ -39,6 +39,11 @@ const (
 	//
 	// Deprecated: legacyDefaultNetworkLabel will be removed in a future release.
 	legacyDefaultNetworkLabel = "capw.vmware.com/is-default-network"
+
+	// AnnotationEnableEndpointHealthCheckKey is the key of the annotation that is used to enable health check on the
+	// Service endpoint port. vm-operator propagates annotations in VMService to Service and LB providers like NSX-T
+	// will enable health check on the endpoint target port when this annotation is present on the Service.
+	AnnotationEnableEndpointHealthCheckKey = "lb.iaas.vmware.com/enable-endpoint-health-check"
 )
 
 var (
@@ -49,5 +54,5 @@ var (
 	NetworkGVKNSXT = ncpv1.SchemeGroupVersion.WithKind("VirtualNetwork")
 
 	// NetworkGVKNSXTVPC is the GVK used for networks in NSX-T VPC mode.
-	NetworkGVKNSXTVPC = nsxopv1.SchemeGroupVersion.WithKind("SubnetSet")
+	NetworkGVKNSXTVPC = nsxvpcv1.SchemeGroupVersion.WithKind("SubnetSet")
 )

pkg/services/network/network_test.go

@@ -24,7 +24,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
@@ -270,7 +270,7 @@ var _ = Describe("Network provider", func() {
 				Expect(vm.Spec.Network.Interfaces).To(HaveLen(1))
 				Expect(vm.Spec.Network.Interfaces[0].Network.Name).To(Equal(vSphereCluster.Name))
 				Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.Kind).To(Equal("SubnetSet"))
-				Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(nsxopv1.SchemeGroupVersion.String()))
+				Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(nsxvpcv1.SchemeGroupVersion.String()))
 			})
 		})
 	})
@@ -336,7 +336,7 @@ var _ = Describe("Network provider", func() {
 			Expect(ncpv1.AddToScheme(scheme)).To(Succeed())
 			Expect(corev1.AddToScheme(scheme)).To(Succeed())
 			Expect(vmwarev1.AddToScheme(scheme)).To(Succeed())
-			Expect(nsxopv1.AddToScheme(scheme)).To(Succeed())
+			Expect(nsxvpcv1.AddToScheme(scheme)).To(Succeed())
 		})
 
 		Context("with dummy network provider", func() {
@@ -599,14 +599,14 @@ var _ = Describe("Network provider", func() {
 
 			It("should not update subnetset", func() {
 				// Fetch the SubnetSet before the operation
-				initialSubnetSet := &nsxopv1.SubnetSet{}
+				initialSubnetSet := &nsxvpcv1.SubnetSet{}
 				err = client.Get(ctx, apitypes.NamespacedName{
 					Name:      dummyCluster,
 					Namespace: dummyNs,
 				}, initialSubnetSet)
 				Expect(err).NotTo(HaveOccurred())
-				status := nsxopv1.SubnetSetStatus{
-					Conditions: []nsxopv1.Condition{
+				status := nsxvpcv1.SubnetSetStatus{
+					Conditions: []nsxvpcv1.Condition{
 						{
 							Type:   "Ready",
 							Status: "True",
@@ -620,7 +620,7 @@ var _ = Describe("Network provider", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name))
 
-				createdSubnetSet := &nsxopv1.SubnetSet{}
+				createdSubnetSet := &nsxvpcv1.SubnetSet{}
 				err = client.Get(ctx, apitypes.NamespacedName{
 					Name:      dummyCluster,
 					Namespace: dummyNs,
@@ -630,10 +630,10 @@ var _ = Describe("Network provider", func() {
 				Expect(createdSubnetSet.Spec).To(Equal(initialSubnetSet.Spec), "SubnetSetSpec should not have been modified")
 			})
 
-			It("should successfully retrieve VM service annotations, confirm they are empty", func() {
+			It("should successfully retrieve VM service annotations, including the annotation to enable LB healthcheck", func() {
 				annotations, err := np.GetVMServiceAnnotations(ctx, clusterCtx)
 				Expect(err).ToNot(HaveOccurred())
-				Expect(annotations).To(Equal(make(map[string]string)))
+				Expect(annotations).To(HaveKey(AnnotationEnableEndpointHealthCheckKey))
 			})
 
 		})
@@ -658,7 +658,7 @@ var _ = Describe("Network provider", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name))
 
-				createdSubnetSet := &nsxopv1.SubnetSet{}
+				createdSubnetSet := &nsxvpcv1.SubnetSet{}
 				err = client.Get(ctx, apitypes.NamespacedName{
 					Name:      dummyCluster,
 					Namespace: dummyNs,
@@ -700,14 +700,14 @@ var _ = Describe("Network provider", func() {
 
 			BeforeEach(func() {
 				scheme = runtime.NewScheme()
-				Expect(nsxopv1.AddToScheme(scheme)).To(Succeed())
+				Expect(nsxvpcv1.AddToScheme(scheme)).To(Succeed())
 				nsxvpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider)
 				np = nsxvpcNp
 			})
 
 			It("should return error when subnetset ready status is false", func() {
-				status := nsxopv1.SubnetSetStatus{
-					Conditions: []nsxopv1.Condition{
+				status := nsxvpcv1.SubnetSetStatus{
+					Conditions: []nsxvpcv1.Condition{
 						{
 							Type:    "Ready",
 							Status:  "False",
@@ -716,7 +716,7 @@ var _ = Describe("Network provider", func() {
 						},
 					},
 				}
-				subnetsetObj = &nsxopv1.SubnetSet{
+				subnetsetObj = &nsxvpcv1.SubnetSet{
 					ObjectMeta: metav1.ObjectMeta{
 						Namespace: cluster.Namespace,
 						Name:      cluster.Name,
@@ -732,10 +732,10 @@ var _ = Describe("Network provider", func() {
 			})
 
 			It("should return error when subnetset ready status is not set", func() {
-				status := nsxopv1.SubnetSetStatus{
-					Conditions: []nsxopv1.Condition{},
+				status := nsxvpcv1.SubnetSetStatus{
+					Conditions: []nsxvpcv1.Condition{},
 				}
-				subnetsetObj = &nsxopv1.SubnetSet{
+				subnetsetObj = &nsxvpcv1.SubnetSet{
 					ObjectMeta: metav1.ObjectMeta{
 						Namespace: cluster.Namespace,
 						Name:      cluster.Name,

pkg/services/network/nsxt_vpc_provider.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
 	vmoprv1common "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common"
 	corev1 "k8s.io/api/core/v1"
@@ -66,13 +66,13 @@ func (vp *nsxtVPCNetworkProvider) SupportsVMReadinessProbe() bool {
 // verifyNsxtVpcSubnetSetStatus checks the status conditions of a given SubnetSet within a cluster context.
 // If the subnet isn't ready, it is marked as false, and the function returns an error.
 // If the subnet is ready, the function updates the VSphereCluster with a "true" status and returns nil.
-func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *nsxopv1.SubnetSet) error {
+func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *nsxvpcv1.SubnetSet) error {
 	clusterName := vspherecluster.Name
 	namespace := vspherecluster.Namespace
 	hasReadyCondition := false
 
 	for _, condition := range subnetset.Status.Conditions {
-		if condition.Type != nsxopv1.Ready {
+		if condition.Type != nsxvpcv1.Ready {
 			continue
 		}
 		hasReadyCondition = true
@@ -96,7 +96,7 @@ func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *v
 // If it is, then it calls verifyNsxVpcSubnetSetStatus with the SubnetSet to verify its status.
 // If it's not, it returns an error.
 func (vp *nsxtVPCNetworkProvider) VerifyNetworkStatus(_ context.Context, clusterCtx *vmware.ClusterContext, obj runtime.Object) error {
-	subnetset, ok := obj.(*nsxopv1.SubnetSet)
+	subnetset, ok := obj.(*nsxvpcv1.SubnetSet)
 	if !ok {
 		return fmt.Errorf("expected NSX VPC SubnetSet but got %T", obj)
 	}
@@ -119,14 +119,14 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c
 	log.Info("Provisioning ")
 	defer log.Info("Finished provisioning")
 
-	subnetset := &nsxopv1.SubnetSet{
+	subnetset := &nsxvpcv1.SubnetSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: networkNamespace,
 			Name:      networkName,
 		},
-		Spec: nsxopv1.SubnetSetSpec{
-			AdvancedConfig: nsxopv1.AdvancedConfig{
-				StaticIPAllocation: nsxopv1.StaticIPAllocation{
+		Spec: nsxvpcv1.SubnetSetSpec{
+			AdvancedConfig: nsxvpcv1.AdvancedConfig{
+				StaticIPAllocation: nsxvpcv1.StaticIPAllocation{
 					Enable: true,
 				},
 			},
@@ -154,7 +154,7 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c
 
 // GetClusterNetworkName returns the name of a valid cluster network if one exists.
 func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clusterCtx *vmware.ClusterContext) (string, error) {
-	subnetset := &nsxopv1.SubnetSet{}
+	subnetset := &nsxvpcv1.SubnetSet{}
 	cluster := clusterCtx.VSphereCluster
 	namespacedName := types.NamespacedName{
 		Namespace: cluster.Namespace,
@@ -168,7 +168,8 @@ func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clu
 
 // The GetVMServiceAnnotations method always returns an empty map representing annotations.
 func (vp *nsxtVPCNetworkProvider) GetVMServiceAnnotations(_ context.Context, _ *vmware.ClusterContext) (map[string]string, error) {
-	return map[string]string{}, nil
+	// The value of the annotation lb.iaas.vmware.com/enable-endpoint-health-check is expected to be an empty string.
+	return map[string]string{AnnotationEnableEndpointHealthCheckKey: ""}, nil
 }
 
 // ConfigureVirtualMachine configures a VirtualMachine object based on the networking configuration.

test/go.sum

@@ -323,8 +323,8 @@ github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXV
 github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:cgx9UH/r53bKU/Gbv8IPsUZ34bj5+ItijA2JCUS3kVk=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea h1:9BE++G23adnANH0J4UCqysUHoryCy678Yra8SCZ39FQ=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
 github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU=
 github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU=