Add NSX-VPC Network Provider Support

This commit introduces NSX-T Virtual Private Cloud (NSX-VPC) as a new network provider for the vSphere Supervisor Cluster in anticipation of the upcoming vSphere 9 release.
Key Changes:
    nsx-vpc implemented as a new NetworkProvider using nsx-operator libs.
    Skipped VM Readiness check as NSX-VPC offers private network access.
    Added unit tests for NSX-VPC network provider."

config/rbac/role.yaml

@@ -334,6 +334,19 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - nsx.vmware.com
+  resources:
+  - subnetsets
+  - subnetsets/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:

controllers/vmware/vspherecluster_reconciler.go

@@ -62,6 +62,7 @@ type ClusterReconciler struct {
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclustertemplates,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.com,resources=virtualnetworks;virtualnetworks/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies;virtualmachinesetresourcepolicies/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices;virtualmachineservices/status,verbs=get;list;watch;create;update;patch;delete

controllers/vspheremachine_controller.go

@@ -90,12 +90,12 @@ func AddMachineControllerToManager(ctx context.Context, controllerManagerContext
 	}
 
 	if supervisorBased {
-		r.VMService = &vmoperator.VmopMachineService{Client: controllerManagerContext.Client}
 		networkProvider, err := inframanager.GetNetworkProvider(ctx, controllerManagerContext.Client, controllerManagerContext.NetworkProvider)
 		if err != nil {
 			return errors.Wrap(err, "failed to create a network provider")
 		}
 		r.networkProvider = networkProvider
+		r.VMService = &vmoperator.VmopMachineService{Client: controllerManagerContext.Client, ConfigureControlPlaneVMReadinessProbe: r.networkProvider.SupportsVMReadinessProbe()}
 
 		return ctrl.NewControllerManagedBy(mgr).
 			// Watch the controlled, infrastructure resource.

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360
+	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0
 	github.com/vmware-tanzu/vm-operator/api v1.8.5
 	github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd
 	github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20231214185006-5477585eebfd

go.sum

@@ -550,6 +550,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360 h1:yG158jviUd3wRqCTJcSDzp+prUZWtSA9dhfm/Rf8m9M=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360/go.mod h1:dtVG693FvGuOSxJvTaKRVGU0EJR8yvLG3E2VaDDHILM=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
 github.com/vmware-tanzu/vm-operator/api v1.8.5 h1:E8rpRdV8+cNp/eNZ/QUHvlrbpPh8uk6bKqwEEmGWe64=
 github.com/vmware-tanzu/vm-operator/api v1.8.5/go.mod h1:SXaSFtnw2502Tzy0bfQVHrvbFDijR96r1ihUYQWPOK8=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd h1:qdfVf7KFW+XX7+D4xC/mlBpRA9+B+opdDPxGdqjxO+4=

pkg/manager/manager.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
+	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1"
@@ -64,6 +65,7 @@ func New(ctx context.Context, opts Options) (Manager, error) {
 	_ = vmoprv1.AddToScheme(opts.Scheme)
 	_ = ncpv1.AddToScheme(opts.Scheme)
 	_ = netopv1.AddToScheme(opts.Scheme)
+	_ = nsxopv1.AddToScheme(opts.Scheme)
 	_ = topologyv1.AddToScheme(opts.Scheme)
 	_ = ipamv1.AddToScheme(opts.Scheme)
 

pkg/manager/network.go

@@ -27,6 +27,8 @@ import (
 )
 
 const (
+	// NSXVPCNetworkProvider identifies the nsx-vpc network provider.
+	NSXVPCNetworkProvider = "NSX-VPC"
 	// NSXNetworkProvider identifies the NSX network provider.
 	NSXNetworkProvider = "NSX"
 	// VDSNetworkProvider identifies the VDS network provider.
@@ -41,6 +43,9 @@ func GetNetworkProvider(ctx context.Context, client client.Client, networkProvid
 	log := ctrl.LoggerFrom(ctx)
 
 	switch networkProvider {
+	case NSXVPCNetworkProvider:
+		log.Info("Pick NSX-VPC network provider")
+		return network.NSXTVpcNetworkProvider(client), nil
 	case NSXNetworkProvider:
 		// TODO: disableFirewall not configurable
 		log.Info("Pick NSX-T network provider")

pkg/services/interfaces.go

@@ -71,6 +71,9 @@ type NetworkProvider interface {
 	// HasLoadBalancer indicates whether this provider has a load balancer for Services.
 	HasLoadBalancer() bool
 
+	// SupportsVMReadinessProbe indicates whether this provider support vm readiness probe.
+	SupportsVMReadinessProbe() bool
+
 	// ProvisionClusterNetwork creates network resource for a given cluster
 	// This operation should be idempotent
 	ProvisionClusterNetwork(ctx context.Context, clusterCtx *vmware.ClusterContext) error

pkg/services/network/constants.go

@@ -22,6 +22,11 @@ const (
 	NSXTTypeNetwork = "nsx-t"
 	// NSXTVNetSelectorKey is also defined in VM Operator.
 	NSXTVNetSelectorKey = "ncp.vmware.com/virtual-network-name"
+	// NSXTVPCSubnetSetNetworkType is the name of the NSX VPC network type. Please refer to:
+	// https://github.com/vmware-tanzu/vm-operator/blob/main/api/v1alpha1/virtualmachine_types.go#L149.
+	NSXTVPCSubnetSetNetworkType = "nsx-t-subnetset"
+	// NSXTVPCSubnetSetSelectorKey has to be aligned with VM Operator design.
+	NSXTVPCSubnetSetSelectorKey = "nsx-operator.vmware.com/subnetset-name"
 
 	// CAPVDefaultNetworkLabel is a label used to identify the default network.
 	CAPVDefaultNetworkLabel = "capv.vmware.com/is-default-network"

pkg/services/network/dummy_provider.go

@@ -38,6 +38,10 @@ func (np *dummyNetworkProvider) HasLoadBalancer() bool {
 	return false
 }
 
+func (np *dummyNetworkProvider) SupportsVMReadinessProbe() bool {
+	return true
+}
+
 func (np *dummyNetworkProvider) ProvisionClusterNetwork(_ context.Context, _ *vmware.ClusterContext) error {
 	return nil
 }

pkg/services/network/netop_provider.go

@@ -48,6 +48,10 @@ func (np *netopNetworkProvider) HasLoadBalancer() bool {
 	return true
 }
 
+func (np *netopNetworkProvider) SupportsVMReadinessProbe() bool {
+	return true
+}
+
 // ProvisionClusterNetwork marks the ClusterNetworkReadyCondition true.
 func (np *netopNetworkProvider) ProvisionClusterNetwork(_ context.Context, clusterCtx *vmware.ClusterContext) error {
 	conditions.MarkTrue(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)

pkg/services/network/network_test.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
+	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
@@ -42,8 +43,8 @@ import (
 
 const (
 	// Mocked virtualnetwork status reason and message.
-	testVnetNotRealizedReason  = "Cannot realize network"
-	testVnetNotRealizedMessage = "NetworkNotRealized"
+	testNetworkNotRealizedReason  = "Cannot realize network"
+	testNetworkNotRealizedMessage = "NetworkNotRealized"
 )
 
 func createUnReadyNsxtVirtualNetwork(ctx *vmware.ClusterContext, status ncpv1.VirtualNetworkStatus) *ncpv1.VirtualNetwork {
@@ -195,13 +196,41 @@ var _ = Describe("Network provider", func() {
 				Expect(vm.Spec.NetworkInterfaces[0].NetworkType).To(Equal("nsx-t"))
 			})
 		})
+
+		Context("with NSX-VPC network provider", func() {
+			BeforeEach(func() {
+				scheme := runtime.NewScheme()
+				Expect(ncpv1.AddToScheme(scheme)).To(Succeed())
+				client := fake.NewClientBuilder().WithScheme(scheme).Build()
+				np = NSXTVpcNetworkProvider(client)
+			})
+
+			It("should add nsx-t-subnetset type network interface", func() {
+				err = np.ConfigureVirtualMachine(ctx, clusterCtx, vm)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(vm.Spec.NetworkInterfaces).To(HaveLen(1))
+			})
+
+			It("nsx-t-subnetset type network interface already exists", func() {
+				err = np.ConfigureVirtualMachine(ctx, clusterCtx, vm)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(vm.Spec.NetworkInterfaces).To(HaveLen(1))
+			})
+
+			AfterEach(func() {
+				Expect(err).ToNot(HaveOccurred())
+				Expect(vm.Spec.NetworkInterfaces[0].NetworkName).To(Equal(vSphereCluster.Name))
+				Expect(vm.Spec.NetworkInterfaces[0].NetworkType).To(Equal("nsx-t-subnetset"))
+			})
+		})
 	})
 
 	Context("ProvisionClusterNetwork", func() {
 		var (
 			scheme             *runtime.Scheme
 			client             runtimeclient.Client
 			nsxNp              *nsxtNetworkProvider
+			vpcNp              *nsxtVPCNetworkProvider
 			runtimeObjs        []runtime.Object
 			vnetObj            runtime.Object
 			configmapObj       runtime.Object
@@ -257,6 +286,7 @@ var _ = Describe("Network provider", func() {
 			Expect(ncpv1.AddToScheme(scheme)).To(Succeed())
 			Expect(corev1.AddToScheme(scheme)).To(Succeed())
 			Expect(vmwarev1.AddToScheme(scheme)).To(Succeed())
+			Expect(nsxopv1.AddToScheme(scheme)).To(Succeed())
 		})
 
 		Context("with dummy network provider", func() {
@@ -459,7 +489,7 @@ var _ = Describe("Network provider", func() {
 				By("create a cluster with virtual network in not ready status")
 				status := ncpv1.VirtualNetworkStatus{
 					Conditions: []ncpv1.VirtualNetworkCondition{
-						{Type: "Ready", Status: "False", Reason: testVnetNotRealizedReason, Message: testVnetNotRealizedMessage},
+						{Type: "Ready", Status: "False", Reason: testNetworkNotRealizedReason, Message: testNetworkNotRealizedMessage},
 					},
 				}
 				vnetObj = createUnReadyNsxtVirtualNetwork(clusterCtx, status)
@@ -470,7 +500,155 @@ var _ = Describe("Network provider", func() {
 				err = np.VerifyNetworkStatus(ctx, clusterCtx, vnetObj)
 
 				expectedErrorMessage := fmt.Sprintf("virtual network ready status is: '%s' in cluster %s. reason: %s, message: %s",
-					"False", apitypes.NamespacedName{Namespace: dummyNs, Name: dummyCluster}, testVnetNotRealizedReason, testVnetNotRealizedMessage)
+					"False", apitypes.NamespacedName{Namespace: dummyNs, Name: dummyCluster}, testNetworkNotRealizedReason, testNetworkNotRealizedMessage)
+				Expect(err).To(MatchError(expectedErrorMessage))
+				Expect(conditions.IsFalse(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)).To(BeTrue())
+			})
+		})
+
+		Context("with NSX-VPC network provider and subnetset exists", func() {
+			BeforeEach(func() {
+				client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(configmapObj, systemNamespaceObj).Build()
+				vpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider)
+				np = vpcNp
+				err = np.ProvisionClusterNetwork(ctx, clusterCtx)
+				clusterCtx.VSphereCluster.Status.Conditions = clusterv1.Conditions{{
+					Type:   vmwarev1.ClusterNetworkReadyCondition,
+					Status: corev1.ConditionTrue,
+				}}
+			})
+
+			It("should not update subnetset", func() {
+				// Fetch the SubnetSet before the operation
+				initialSubnetSet := &nsxopv1.SubnetSet{}
+				err = client.Get(ctx, apitypes.NamespacedName{
+					Name:      dummyCluster,
+					Namespace: dummyNs,
+				}, initialSubnetSet)
+				status := nsxopv1.SubnetSetStatus{
+					Conditions: []nsxopv1.Condition{
+						{
+							Type:   "Ready",
+							Status: "True",
+						},
+					},
+				}
+				initialSubnetSet.Status = status
+				Expect(err).NotTo(HaveOccurred())
+
+				// Presumably there's code here that might modify the SubnetSet...
+				Expect(err).ToNot(HaveOccurred())
+				subnetset, localerr := np.GetClusterNetworkName(ctx, clusterCtx)
+				Expect(localerr).ToNot(HaveOccurred())
+				Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name))
+
+				createdSubnetSet := &nsxopv1.SubnetSet{}
+				err = client.Get(ctx, apitypes.NamespacedName{
+					Name:      dummyCluster,
+					Namespace: dummyNs,
+				}, createdSubnetSet)
+
+				Expect(err).ToNot(HaveOccurred())
+				// Check that the SubnetSetSpec was not changed
+				Expect(createdSubnetSet.Spec).To(Equal(initialSubnetSet.Spec), "SubnetSetSpec should not have been modified")
+			})
+
+			It("should successfully retrieve VM service annotations and validate their content and network readiness", func() {
+				annotations, err := np.GetVMServiceAnnotations(ctx, clusterCtx)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(annotations).To(HaveKeyWithValue(NSXTVPCSubnetSetSelectorKey, clusterCtx.VSphereCluster.Name))
+				Expect(conditions.IsTrue(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)).To(BeTrue())
+			})
+		})
+
+		Context("with NSX-VPC network provider and subnetset does not exist", func() {
+			var nsxvpcNp *nsxtVPCNetworkProvider
+
+			BeforeEach(func() {
+				client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(configmapObj, systemNamespaceObj).Build()
+				nsxvpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider)
+				np = nsxvpcNp
+				err = np.ProvisionClusterNetwork(ctx, clusterCtx)
+			})
+
+			It("should create subnetset with new spec", func() {
+				subnetset, localerr := np.GetClusterNetworkName(ctx, clusterCtx)
+				Expect(localerr).ToNot(HaveOccurred())
+				Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name))
+
+				createdSubnetSet := &nsxopv1.SubnetSet{}
+				err = client.Get(ctx, apitypes.NamespacedName{
+					Name:      dummyCluster,
+					Namespace: dummyNs,
+				}, createdSubnetSet)
+
+				Expect(err).ToNot(HaveOccurred())
+				Expect(createdSubnetSet.Spec.AdvancedConfig.StaticIPAllocation.Enable).To(BeTrue())
+			})
+		})
+
+		Context("with NSX-VPC network provider and object passed to VerifyNetworkStatus is not a SubnetSet", func() {
+			var nsxvpcNp *nsxtVPCNetworkProvider
+
+			BeforeEach(func() {
+				client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(configmapObj, systemNamespaceObj).Build()
+				nsxvpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider)
+				np = nsxvpcNp
+			})
+
+			It("should return error when non-SubnetSet object passed to VerifyNetworkStatus", func() {
+				dummyObj := &ncpv1.VirtualNetwork{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: cluster.Namespace,
+						Name:      GetNSXTVirtualNetworkName(cluster.Name),
+					},
+				}
+				err = nsxvpcNp.VerifyNetworkStatus(ctx, clusterCtx, dummyObj)
+				Expect(err).To(HaveOccurred()) // Expect error because dummyObj is not a SubnetSet
+				Expect(err.Error()).To(Equal(fmt.Sprintf("expected NSX VPC SubnetSet but got %T", dummyObj)))
+			})
+		})
+
+		Context("with NSX-VPC network provider failure", func() {
+			var (
+				client       runtimeclient.Client
+				nsxvpcNp     *nsxtVPCNetworkProvider
+				scheme       *runtime.Scheme
+				subnetsetObj runtime.Object
+			)
+
+			BeforeEach(func() {
+				scheme = runtime.NewScheme()
+				Expect(nsxopv1.AddToScheme(scheme)).To(Succeed())
+			})
+
+			It("should return error when subnetset ready status is false", func() {
+				By("create a cluster with subnetset in not ready status")
+				status := nsxopv1.SubnetSetStatus{
+					Conditions: []nsxopv1.Condition{
+						{
+							Type:    "Ready",
+							Status:  "False",
+							Reason:  testNetworkNotRealizedReason,
+							Message: testNetworkNotRealizedMessage,
+						},
+					},
+				}
+				subnetsetObj = &nsxopv1.SubnetSet{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: cluster.Namespace,
+						Name:      cluster.Name,
+					},
+					Status: status,
+				}
+				client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(subnetsetObj).Build()
+				nsxvpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider)
+				np = nsxvpcNp
+
+				err = np.VerifyNetworkStatus(ctx, clusterCtx, subnetsetObj)
+
+				expectedErrorMessage := fmt.Sprintf("subnetset ready status is: '%s' in cluster %s. reason: %s, message: %s",
+					"False", apitypes.NamespacedName{Namespace: dummyNs, Name: dummyCluster}, testNetworkNotRealizedReason, testNetworkNotRealizedMessage)
 				Expect(err).To(MatchError(expectedErrorMessage))
 				Expect(conditions.IsFalse(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)).To(BeTrue())
 			})

pkg/services/network/nsxt_provider.go

@@ -56,6 +56,10 @@ func (np *nsxtNetworkProvider) HasLoadBalancer() bool {
 	return true
 }
 
+func (np *nsxtNetworkProvider) SupportsVMReadinessProbe() bool {
+	return true
+}
+
 // GetNSXTVirtualNetworkName returns the name of the NSX-T vnet object.
 func GetNSXTVirtualNetworkName(clusterName string) string {
 	return fmt.Sprintf("%s-vnet", clusterName)