✨ Enable NSX LB healthcheck on kube-apiserver

.golangci.yml

@@ -122,8 +122,11 @@ linters-settings:
         alias: vmwarev1
       # VMware Operator
       - pkg: "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
-        alias: vmoprv1 
-      # CABPK
+        alias: vmoprv1
+      # NSX Operator
+      - pkg: "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
+        alias: nsxvpcv1
+    # CABPK
       - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1
         alias: bootstrapv1
       # KCP

config/rbac/role.yaml

@@ -187,6 +187,19 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - crd.nsx.vmware.com
+  resources:
+  - subnetsets
+  - subnetsets/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - infrastructure.cluster.x-k8s.io
   resources:
@@ -334,19 +347,6 @@ rules:
   - get
   - list
   - watch
-- apiGroups:
-  - nsx.vmware.com
-  resources:
-  - subnetsets
-  - subnetsets/status
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:

controllers/vmware/vspherecluster_reconciler.go

@@ -62,7 +62,7 @@ type ClusterReconciler struct {
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclustertemplates,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=crd.nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.com,resources=virtualnetworks;virtualnetworks/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies;virtualmachinesetresourcepolicies/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices;virtualmachineservices/status,verbs=get;list;watch;create;update;patch;delete

go.mod

@@ -8,7 +8,7 @@ replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.c
 
 require (
 	github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d
-	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0
+	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea
 	// The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-tests
 	github.com/vmware-tanzu/vm-operator/api v1.8.6
 	github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505

go.sum

@@ -270,8 +270,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:cgx9UH/r53bKU/Gbv8IPsUZ34bj5+ItijA2JCUS3kVk=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea h1:9BE++G23adnANH0J4UCqysUHoryCy678Yra8SCZ39FQ=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
 github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU=
 github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU=

pkg/manager/manager.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	"gopkg.in/fsnotify.v1"
@@ -65,7 +65,7 @@ func New(ctx context.Context, opts Options) (Manager, error) {
 	_ = vmoprv1.AddToScheme(opts.Scheme)
 	_ = ncpv1.AddToScheme(opts.Scheme)
 	_ = netopv1.AddToScheme(opts.Scheme)
-	_ = nsxopv1.AddToScheme(opts.Scheme)
+	_ = nsxvpcv1.AddToScheme(opts.Scheme)
 	_ = topologyv1.AddToScheme(opts.Scheme)
 	_ = ipamv1.AddToScheme(opts.Scheme)
 

pkg/services/network/constants.go

@@ -19,7 +19,7 @@ package network
 
 import (
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 )
 
@@ -39,6 +39,11 @@ const (
 	//
 	// Deprecated: legacyDefaultNetworkLabel will be removed in a future release.
 	legacyDefaultNetworkLabel = "capw.vmware.com/is-default-network"
+
+	// AnnotationEnableEndpointHealthCheckKey is the key of the annotation that is used to enable health check on the
+	// Service endpoint port. vm-operator propagates annotations in VMService to Service and LB providers like NSX-T
+	// will enable health check on the endpoint target port when this annotation is present on the Service.
+	AnnotationEnableEndpointHealthCheckKey = "lb.iaas.vmware.com/enable-endpoint-health-check"
 )
 
 var (
@@ -49,5 +54,5 @@ var (
 	NetworkGVKNSXT = ncpv1.SchemeGroupVersion.WithKind("VirtualNetwork")
 
 	// NetworkGVKNSXTVPC is the GVK used for networks in NSX-T VPC mode.
-	NetworkGVKNSXTVPC = nsxopv1.SchemeGroupVersion.WithKind("SubnetSet")
+	NetworkGVKNSXTVPC = nsxvpcv1.SchemeGroupVersion.WithKind("SubnetSet")
 )

pkg/services/network/network_test.go

@@ -24,7 +24,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
@@ -270,7 +270,7 @@ var _ = Describe("Network provider", func() {
 				Expect(vm.Spec.Network.Interfaces).To(HaveLen(1))
 				Expect(vm.Spec.Network.Interfaces[0].Network.Name).To(Equal(vSphereCluster.Name))
 				Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.Kind).To(Equal("SubnetSet"))
-				Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(nsxopv1.SchemeGroupVersion.String()))
+				Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(nsxvpcv1.SchemeGroupVersion.String()))
 			})
 		})
 	})
@@ -336,7 +336,7 @@ var _ = Describe("Network provider", func() {
 			Expect(ncpv1.AddToScheme(scheme)).To(Succeed())
 			Expect(corev1.AddToScheme(scheme)).To(Succeed())
 			Expect(vmwarev1.AddToScheme(scheme)).To(Succeed())
-			Expect(nsxopv1.AddToScheme(scheme)).To(Succeed())
+			Expect(nsxvpcv1.AddToScheme(scheme)).To(Succeed())
 		})
 
 		Context("with dummy network provider", func() {
@@ -599,14 +599,14 @@ var _ = Describe("Network provider", func() {
 
 			It("should not update subnetset", func() {
 				// Fetch the SubnetSet before the operation
-				initialSubnetSet := &nsxopv1.SubnetSet{}
+				initialSubnetSet := &nsxvpcv1.SubnetSet{}
 				err = client.Get(ctx, apitypes.NamespacedName{
 					Name:      dummyCluster,
 					Namespace: dummyNs,
 				}, initialSubnetSet)
 				Expect(err).NotTo(HaveOccurred())
-				status := nsxopv1.SubnetSetStatus{
-					Conditions: []nsxopv1.Condition{
+				status := nsxvpcv1.SubnetSetStatus{
+					Conditions: []nsxvpcv1.Condition{
 						{
 							Type:   "Ready",
 							Status: "True",
@@ -620,7 +620,7 @@ var _ = Describe("Network provider", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name))
 
-				createdSubnetSet := &nsxopv1.SubnetSet{}
+				createdSubnetSet := &nsxvpcv1.SubnetSet{}
 				err = client.Get(ctx, apitypes.NamespacedName{
 					Name:      dummyCluster,
 					Namespace: dummyNs,
@@ -630,10 +630,10 @@ var _ = Describe("Network provider", func() {
 				Expect(createdSubnetSet.Spec).To(Equal(initialSubnetSet.Spec), "SubnetSetSpec should not have been modified")
 			})
 
-			It("should successfully retrieve VM service annotations, confirm they are empty", func() {
+			It("should successfully retrieve VM service annotations, including the annotation to enable LB healthcheck", func() {
 				annotations, err := np.GetVMServiceAnnotations(ctx, clusterCtx)
 				Expect(err).ToNot(HaveOccurred())
-				Expect(annotations).To(Equal(make(map[string]string)))
+				Expect(annotations).To(HaveKey(AnnotationEnableEndpointHealthCheckKey))
 			})
 
 		})
@@ -658,7 +658,7 @@ var _ = Describe("Network provider", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name))
 
-				createdSubnetSet := &nsxopv1.SubnetSet{}
+				createdSubnetSet := &nsxvpcv1.SubnetSet{}
 				err = client.Get(ctx, apitypes.NamespacedName{
 					Name:      dummyCluster,
 					Namespace: dummyNs,
@@ -700,14 +700,14 @@ var _ = Describe("Network provider", func() {
 
 			BeforeEach(func() {
 				scheme = runtime.NewScheme()
-				Expect(nsxopv1.AddToScheme(scheme)).To(Succeed())
+				Expect(nsxvpcv1.AddToScheme(scheme)).To(Succeed())
 				nsxvpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider)
 				np = nsxvpcNp
 			})
 
 			It("should return error when subnetset ready status is false", func() {
-				status := nsxopv1.SubnetSetStatus{
-					Conditions: []nsxopv1.Condition{
+				status := nsxvpcv1.SubnetSetStatus{
+					Conditions: []nsxvpcv1.Condition{
 						{
 							Type:    "Ready",
 							Status:  "False",
@@ -716,7 +716,7 @@ var _ = Describe("Network provider", func() {
 						},
 					},
 				}
-				subnetsetObj = &nsxopv1.SubnetSet{
+				subnetsetObj = &nsxvpcv1.SubnetSet{
 					ObjectMeta: metav1.ObjectMeta{
 						Namespace: cluster.Namespace,
 						Name:      cluster.Name,
@@ -732,10 +732,10 @@ var _ = Describe("Network provider", func() {
 			})
 
 			It("should return error when subnetset ready status is not set", func() {
-				status := nsxopv1.SubnetSetStatus{
-					Conditions: []nsxopv1.Condition{},
+				status := nsxvpcv1.SubnetSetStatus{
+					Conditions: []nsxvpcv1.Condition{},
 				}
-				subnetsetObj = &nsxopv1.SubnetSet{
+				subnetsetObj = &nsxvpcv1.SubnetSet{
 					ObjectMeta: metav1.ObjectMeta{
 						Namespace: cluster.Namespace,
 						Name:      cluster.Name,

pkg/services/network/nsxt_vpc_provider.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
-	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+	nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
 	vmoprv1common "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common"
 	corev1 "k8s.io/api/core/v1"
@@ -66,13 +66,13 @@ func (vp *nsxtVPCNetworkProvider) SupportsVMReadinessProbe() bool {
 // verifyNsxtVpcSubnetSetStatus checks the status conditions of a given SubnetSet within a cluster context.
 // If the subnet isn't ready, it is marked as false, and the function returns an error.
 // If the subnet is ready, the function updates the VSphereCluster with a "true" status and returns nil.
-func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *nsxopv1.SubnetSet) error {
+func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *nsxvpcv1.SubnetSet) error {
 	clusterName := vspherecluster.Name
 	namespace := vspherecluster.Namespace
 	hasReadyCondition := false
 
 	for _, condition := range subnetset.Status.Conditions {
-		if condition.Type != nsxopv1.Ready {
+		if condition.Type != nsxvpcv1.Ready {
 			continue
 		}
 		hasReadyCondition = true
@@ -96,7 +96,7 @@ func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *v
 // If it is, then it calls verifyNsxVpcSubnetSetStatus with the SubnetSet to verify its status.
 // If it's not, it returns an error.
 func (vp *nsxtVPCNetworkProvider) VerifyNetworkStatus(_ context.Context, clusterCtx *vmware.ClusterContext, obj runtime.Object) error {
-	subnetset, ok := obj.(*nsxopv1.SubnetSet)
+	subnetset, ok := obj.(*nsxvpcv1.SubnetSet)
 	if !ok {
 		return fmt.Errorf("expected NSX VPC SubnetSet but got %T", obj)
 	}
@@ -119,14 +119,14 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c
 	log.Info("Provisioning ")
 	defer log.Info("Finished provisioning")
 
-	subnetset := &nsxopv1.SubnetSet{
+	subnetset := &nsxvpcv1.SubnetSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: networkNamespace,
 			Name:      networkName,
 		},
-		Spec: nsxopv1.SubnetSetSpec{
-			AdvancedConfig: nsxopv1.AdvancedConfig{
-				StaticIPAllocation: nsxopv1.StaticIPAllocation{
+		Spec: nsxvpcv1.SubnetSetSpec{
+			AdvancedConfig: nsxvpcv1.AdvancedConfig{
+				StaticIPAllocation: nsxvpcv1.StaticIPAllocation{
 					Enable: true,
 				},
 			},
@@ -154,7 +154,7 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c
 
 // GetClusterNetworkName returns the name of a valid cluster network if one exists.
 func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clusterCtx *vmware.ClusterContext) (string, error) {
-	subnetset := &nsxopv1.SubnetSet{}
+	subnetset := &nsxvpcv1.SubnetSet{}
 	cluster := clusterCtx.VSphereCluster
 	namespacedName := types.NamespacedName{
 		Namespace: cluster.Namespace,
@@ -168,7 +168,8 @@ func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clu
 
 // The GetVMServiceAnnotations method always returns an empty map representing annotations.
 func (vp *nsxtVPCNetworkProvider) GetVMServiceAnnotations(_ context.Context, _ *vmware.ClusterContext) (map[string]string, error) {
-	return map[string]string{}, nil
+	// The value of the annotation lb.iaas.vmware.com/enable-endpoint-health-check is expected to be an empty string.
+	return map[string]string{AnnotationEnableEndpointHealthCheckKey: ""}, nil
 }
 
 // ConfigureVirtualMachine configures a VirtualMachine object based on the networking configuration.

test/go.sum

@@ -323,8 +323,8 @@ github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXV
 github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:cgx9UH/r53bKU/Gbv8IPsUZ34bj5+ItijA2JCUS3kVk=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4=
-github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea h1:9BE++G23adnANH0J4UCqysUHoryCy678Yra8SCZ39FQ=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
 github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU=
 github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU=