Add NSX-VPC Network Provider Support

This commit introduces NSX-T Virtual Private Cloud (NSX-VPC) as a new network provider for the vSphere Supervisor Cluster in anticipation of the upcoming vSphere 9 release.
Key Changes:
    nsx-vpc implemented as a new NetworkProvider using nsx-operator libs.
    Skipped VM Readiness check as NSX-VPC offers private network access.
    Added unit tests for NSX-VPC network provider."

config/rbac/role.yaml

@@ -334,6 +334,19 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - nsx.vmware.com
+  resources:
+  - subnetsets
+  - subnetsets/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:

controllers/vmware/vspherecluster_reconciler.go

@@ -62,6 +62,7 @@ type ClusterReconciler struct {
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclustertemplates,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmware.com,resources=virtualnetworks;virtualnetworks/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies;virtualmachinesetresourcepolicies/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices;virtualmachineservices/status,verbs=get;list;watch;create;update;patch;delete

controllers/vspheremachine_controller.go

@@ -90,12 +90,12 @@ func AddMachineControllerToManager(ctx context.Context, controllerManagerContext
 	}
 
 	if supervisorBased {
-		r.VMService = &vmoperator.VmopMachineService{Client: controllerManagerContext.Client}
 		networkProvider, err := inframanager.GetNetworkProvider(ctx, controllerManagerContext.Client, controllerManagerContext.NetworkProvider)
 		if err != nil {
 			return errors.Wrap(err, "failed to create a network provider")
 		}
 		r.networkProvider = networkProvider
+		r.VMService = &vmoperator.VmopMachineService{Client: controllerManagerContext.Client, ConfigureControlPlaneVMReadinessProbe: r.networkProvider.SupportsVMReadinessProbe()}
 
 		return ctrl.NewControllerManagedBy(mgr).
 			// Watch the controlled, infrastructure resource.

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360
+	github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0
 	github.com/vmware-tanzu/vm-operator/api v1.8.5
 	github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd
 	github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20231214185006-5477585eebfd

go.sum

@@ -550,6 +550,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360 h1:yG158jviUd3wRqCTJcSDzp+prUZWtSA9dhfm/Rf8m9M=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360/go.mod h1:dtVG693FvGuOSxJvTaKRVGU0EJR8yvLG3E2VaDDHILM=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4=
+github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk=
 github.com/vmware-tanzu/vm-operator/api v1.8.5 h1:E8rpRdV8+cNp/eNZ/QUHvlrbpPh8uk6bKqwEEmGWe64=
 github.com/vmware-tanzu/vm-operator/api v1.8.5/go.mod h1:SXaSFtnw2502Tzy0bfQVHrvbFDijR96r1ihUYQWPOK8=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd h1:qdfVf7KFW+XX7+D4xC/mlBpRA9+B+opdDPxGdqjxO+4=

pkg/manager/manager.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1"
+	nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1"
@@ -64,6 +65,7 @@ func New(ctx context.Context, opts Options) (Manager, error) {
 	_ = vmoprv1.AddToScheme(opts.Scheme)
 	_ = ncpv1.AddToScheme(opts.Scheme)
 	_ = netopv1.AddToScheme(opts.Scheme)
+	_ = nsxopv1.AddToScheme(opts.Scheme)
 	_ = topologyv1.AddToScheme(opts.Scheme)
 	_ = ipamv1.AddToScheme(opts.Scheme)
 

pkg/manager/network.go

@@ -27,6 +27,8 @@ import (
 )
 
 const (
+	// NSXVPCNetworkProvider identifies the nsx-vpc network provider.
+	NSXVPCNetworkProvider = "NSX-VPC"
 	// NSXNetworkProvider identifies the NSX network provider.
 	NSXNetworkProvider = "NSX"
 	// VDSNetworkProvider identifies the VDS network provider.
@@ -41,6 +43,9 @@ func GetNetworkProvider(ctx context.Context, client client.Client, networkProvid
 	log := ctrl.LoggerFrom(ctx)
 
 	switch networkProvider {
+	case NSXVPCNetworkProvider:
+		log.Info("Pick NSX-VPC network provider")
+		return network.NSXTVpcNetworkProvider(client), nil
 	case NSXNetworkProvider:
 		// TODO: disableFirewall not configurable
 		log.Info("Pick NSX-T network provider")

pkg/services/interfaces.go

@@ -71,6 +71,9 @@ type NetworkProvider interface {
 	// HasLoadBalancer indicates whether this provider has a load balancer for Services.
 	HasLoadBalancer() bool
 
+	// SupportsVMReadinessProbe indicates whether this provider support vm readiness probe.
+	SupportsVMReadinessProbe() bool
+
 	// ProvisionClusterNetwork creates network resource for a given cluster
 	// This operation should be idempotent
 	ProvisionClusterNetwork(ctx context.Context, clusterCtx *vmware.ClusterContext) error

pkg/services/network/constants.go

@@ -22,6 +22,9 @@ const (
 	NSXTTypeNetwork = "nsx-t"
 	// NSXTVNetSelectorKey is also defined in VM Operator.
 	NSXTVNetSelectorKey = "ncp.vmware.com/virtual-network-name"
+	// NSXTVPCSubnetSetNetworkType is the name of the NSX VPC network type. Please refer to:
+	// https://github.com/vmware-tanzu/vm-operator/blob/main/api/v1alpha1/virtualmachine_types.go#L149.
+	NSXTVPCSubnetSetNetworkType = "nsx-t-subnetset"
 
 	// CAPVDefaultNetworkLabel is a label used to identify the default network.
 	CAPVDefaultNetworkLabel = "capv.vmware.com/is-default-network"

pkg/services/network/dummy_provider.go

@@ -38,6 +38,10 @@ func (np *dummyNetworkProvider) HasLoadBalancer() bool {
 	return false
 }
 
+func (np *dummyNetworkProvider) SupportsVMReadinessProbe() bool {
+	return true
+}
+
 func (np *dummyNetworkProvider) ProvisionClusterNetwork(_ context.Context, _ *vmware.ClusterContext) error {
 	return nil
 }

pkg/services/network/netop_provider.go

@@ -48,6 +48,10 @@ func (np *netopNetworkProvider) HasLoadBalancer() bool {
 	return true
 }
 
+func (np *netopNetworkProvider) SupportsVMReadinessProbe() bool {
+	return true
+}
+
 // ProvisionClusterNetwork marks the ClusterNetworkReadyCondition true.
 func (np *netopNetworkProvider) ProvisionClusterNetwork(_ context.Context, clusterCtx *vmware.ClusterContext) error {
 	conditions.MarkTrue(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)