From 85e8cab613e5803195ed82912f5941bbb94c0075 Mon Sep 17 00:00:00 2001 From: Yifeng Xiao Date: Wed, 31 Jul 2024 14:55:54 +0800 Subject: [PATCH 1/3] Enable NSX LB healthcheck on kube-apiserver In the VPC network, the vm-operator Pod and Kubernetes Clusters are in different VPCs. TCP readiness probe does not work across VPC. Use LB 's healthcheck instead. This annotation is agnostic to LB providers. NSX LB supports it to enable healthcheck while AVI LB has healthcheck enabled by default. --- pkg/services/network/network_test.go | 4 ++-- pkg/services/network/nsxt_vpc_provider.go | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/services/network/network_test.go b/pkg/services/network/network_test.go index 33114c6e1f..72c025738b 100644 --- a/pkg/services/network/network_test.go +++ b/pkg/services/network/network_test.go @@ -630,10 +630,10 @@ var _ = Describe("Network provider", func() { Expect(createdSubnetSet.Spec).To(Equal(initialSubnetSet.Spec), "SubnetSetSpec should not have been modified") }) - It("should successfully retrieve VM service annotations, confirm they are empty", func() { + It("should successfully retrieve VM service annotations, including the annotation to enable LB healthcheck", func() { annotations, err := np.GetVMServiceAnnotations(ctx, clusterCtx) Expect(err).ToNot(HaveOccurred()) - Expect(annotations).To(Equal(make(map[string]string))) + Expect(annotations).To(HaveKey("lb.iaas.vmware.com/enable-endpoint-health-check")) }) }) diff --git a/pkg/services/network/nsxt_vpc_provider.go b/pkg/services/network/nsxt_vpc_provider.go index b9fd24349d..0eae9e5bb6 100644 --- a/pkg/services/network/nsxt_vpc_provider.go +++ b/pkg/services/network/nsxt_vpc_provider.go @@ -40,6 +40,11 @@ import ( "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services" ) +const ( + // AnnotationEnableEndpointHealthCheckKey is the key of the annotation that is used to enable health check on the VMService endpoint port. + AnnotationEnableEndpointHealthCheckKey = "lb.iaas.vmware.com/enable-endpoint-health-check" +) + // nsxtVPCNetworkProvider provisions nsx-vpc type cluster network. type nsxtVPCNetworkProvider struct { client client.Client @@ -168,7 +173,7 @@ func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clu // The GetVMServiceAnnotations method always returns an empty map representing annotations. func (vp *nsxtVPCNetworkProvider) GetVMServiceAnnotations(_ context.Context, _ *vmware.ClusterContext) (map[string]string, error) { - return map[string]string{}, nil + return map[string]string{AnnotationEnableEndpointHealthCheckKey: ""}, nil } // ConfigureVirtualMachine configures a VirtualMachine object based on the networking configuration. From a409446f3728a0bd65236ae420020558d990c9be Mon Sep 17 00:00:00 2001 From: Yifeng Xiao Date: Wed, 14 Aug 2024 16:56:14 +0800 Subject: [PATCH 2/3] Update nsx-operator API mod There is a group refactor in nsx-operator API, so we need to update to latest version. --- config/rbac/role.yaml | 26 ++++++++-------- .../vmware/vspherecluster_reconciler.go | 2 +- go.mod | 2 +- go.sum | 4 +-- pkg/manager/manager.go | 4 +-- pkg/services/network/constants.go | 4 +-- pkg/services/network/network_test.go | 30 +++++++++---------- pkg/services/network/nsxt_vpc_provider.go | 18 +++++------ test/go.sum | 4 +-- 9 files changed, 47 insertions(+), 47 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 593899b095..0a1a04a20b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -187,6 +187,19 @@ rules: - patch - update - watch +- apiGroups: + - crd.nsx.vmware.com + resources: + - subnetsets + - subnetsets/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - infrastructure.cluster.x-k8s.io resources: @@ -334,19 +347,6 @@ rules: - get - list - watch -- apiGroups: - - nsx.vmware.com - resources: - - subnetsets - - subnetsets/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/controllers/vmware/vspherecluster_reconciler.go b/controllers/vmware/vspherecluster_reconciler.go index d558c8877c..397663de19 100644 --- a/controllers/vmware/vspherecluster_reconciler.go +++ b/controllers/vmware/vspherecluster_reconciler.go @@ -62,7 +62,7 @@ type ClusterReconciler struct { // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclustertemplates,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crd.nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmware.com,resources=virtualnetworks;virtualnetworks/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies;virtualmachinesetresourcepolicies/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices;virtualmachineservices/status,verbs=get;list;watch;create;update;patch;delete diff --git a/go.mod b/go.mod index 945ea5df08..61cba0380e 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.c require ( github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d - github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 + github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea // The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-tests github.com/vmware-tanzu/vm-operator/api v1.8.6 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 diff --git a/go.sum b/go.sum index bfe27b1238..6d6899d0b2 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:cgx9UH/r53bKU/Gbv8IPsUZ34bj5+ItijA2JCUS3kVk= github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM= -github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4= -github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea h1:9BE++G23adnANH0J4UCqysUHoryCy678Yra8SCZ39FQ= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU= github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU= diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index c17a811aea..ae5f36f269 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -22,7 +22,7 @@ import ( "github.com/pkg/errors" netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" - nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" "gopkg.in/fsnotify.v1" @@ -65,7 +65,7 @@ func New(ctx context.Context, opts Options) (Manager, error) { _ = vmoprv1.AddToScheme(opts.Scheme) _ = ncpv1.AddToScheme(opts.Scheme) _ = netopv1.AddToScheme(opts.Scheme) - _ = nsxopv1.AddToScheme(opts.Scheme) + _ = vpcapisv1.AddToScheme(opts.Scheme) _ = topologyv1.AddToScheme(opts.Scheme) _ = ipamv1.AddToScheme(opts.Scheme) diff --git a/pkg/services/network/constants.go b/pkg/services/network/constants.go index 902c8d24ce..76058840b9 100644 --- a/pkg/services/network/constants.go +++ b/pkg/services/network/constants.go @@ -19,7 +19,7 @@ package network import ( netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" - nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" ) @@ -49,5 +49,5 @@ var ( NetworkGVKNSXT = ncpv1.SchemeGroupVersion.WithKind("VirtualNetwork") // NetworkGVKNSXTVPC is the GVK used for networks in NSX-T VPC mode. - NetworkGVKNSXTVPC = nsxopv1.SchemeGroupVersion.WithKind("SubnetSet") + NetworkGVKNSXTVPC = vpcapisv1.SchemeGroupVersion.WithKind("SubnetSet") ) diff --git a/pkg/services/network/network_test.go b/pkg/services/network/network_test.go index 72c025738b..6b26c112ef 100644 --- a/pkg/services/network/network_test.go +++ b/pkg/services/network/network_test.go @@ -24,7 +24,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" - nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -270,7 +270,7 @@ var _ = Describe("Network provider", func() { Expect(vm.Spec.Network.Interfaces).To(HaveLen(1)) Expect(vm.Spec.Network.Interfaces[0].Network.Name).To(Equal(vSphereCluster.Name)) Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.Kind).To(Equal("SubnetSet")) - Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(nsxopv1.SchemeGroupVersion.String())) + Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(vpcapisv1.SchemeGroupVersion.String())) }) }) }) @@ -336,7 +336,7 @@ var _ = Describe("Network provider", func() { Expect(ncpv1.AddToScheme(scheme)).To(Succeed()) Expect(corev1.AddToScheme(scheme)).To(Succeed()) Expect(vmwarev1.AddToScheme(scheme)).To(Succeed()) - Expect(nsxopv1.AddToScheme(scheme)).To(Succeed()) + Expect(vpcapisv1.AddToScheme(scheme)).To(Succeed()) }) Context("with dummy network provider", func() { @@ -599,14 +599,14 @@ var _ = Describe("Network provider", func() { It("should not update subnetset", func() { // Fetch the SubnetSet before the operation - initialSubnetSet := &nsxopv1.SubnetSet{} + initialSubnetSet := &vpcapisv1.SubnetSet{} err = client.Get(ctx, apitypes.NamespacedName{ Name: dummyCluster, Namespace: dummyNs, }, initialSubnetSet) Expect(err).NotTo(HaveOccurred()) - status := nsxopv1.SubnetSetStatus{ - Conditions: []nsxopv1.Condition{ + status := vpcapisv1.SubnetSetStatus{ + Conditions: []vpcapisv1.Condition{ { Type: "Ready", Status: "True", @@ -620,7 +620,7 @@ var _ = Describe("Network provider", func() { Expect(err).ToNot(HaveOccurred()) Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name)) - createdSubnetSet := &nsxopv1.SubnetSet{} + createdSubnetSet := &vpcapisv1.SubnetSet{} err = client.Get(ctx, apitypes.NamespacedName{ Name: dummyCluster, Namespace: dummyNs, @@ -658,7 +658,7 @@ var _ = Describe("Network provider", func() { Expect(err).ToNot(HaveOccurred()) Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name)) - createdSubnetSet := &nsxopv1.SubnetSet{} + createdSubnetSet := &vpcapisv1.SubnetSet{} err = client.Get(ctx, apitypes.NamespacedName{ Name: dummyCluster, Namespace: dummyNs, @@ -700,14 +700,14 @@ var _ = Describe("Network provider", func() { BeforeEach(func() { scheme = runtime.NewScheme() - Expect(nsxopv1.AddToScheme(scheme)).To(Succeed()) + Expect(vpcapisv1.AddToScheme(scheme)).To(Succeed()) nsxvpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider) np = nsxvpcNp }) It("should return error when subnetset ready status is false", func() { - status := nsxopv1.SubnetSetStatus{ - Conditions: []nsxopv1.Condition{ + status := vpcapisv1.SubnetSetStatus{ + Conditions: []vpcapisv1.Condition{ { Type: "Ready", Status: "False", @@ -716,7 +716,7 @@ var _ = Describe("Network provider", func() { }, }, } - subnetsetObj = &nsxopv1.SubnetSet{ + subnetsetObj = &vpcapisv1.SubnetSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, Name: cluster.Name, @@ -732,10 +732,10 @@ var _ = Describe("Network provider", func() { }) It("should return error when subnetset ready status is not set", func() { - status := nsxopv1.SubnetSetStatus{ - Conditions: []nsxopv1.Condition{}, + status := vpcapisv1.SubnetSetStatus{ + Conditions: []vpcapisv1.Condition{}, } - subnetsetObj = &nsxopv1.SubnetSet{ + subnetsetObj = &vpcapisv1.SubnetSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, Name: cluster.Name, diff --git a/pkg/services/network/nsxt_vpc_provider.go b/pkg/services/network/nsxt_vpc_provider.go index 0eae9e5bb6..f4251b9c79 100644 --- a/pkg/services/network/nsxt_vpc_provider.go +++ b/pkg/services/network/nsxt_vpc_provider.go @@ -21,7 +21,7 @@ import ( "fmt" "github.com/pkg/errors" - nsxopv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" vmoprv1common "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" corev1 "k8s.io/api/core/v1" @@ -71,13 +71,13 @@ func (vp *nsxtVPCNetworkProvider) SupportsVMReadinessProbe() bool { // verifyNsxtVpcSubnetSetStatus checks the status conditions of a given SubnetSet within a cluster context. // If the subnet isn't ready, it is marked as false, and the function returns an error. // If the subnet is ready, the function updates the VSphereCluster with a "true" status and returns nil. -func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *nsxopv1.SubnetSet) error { +func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *vpcapisv1.SubnetSet) error { clusterName := vspherecluster.Name namespace := vspherecluster.Namespace hasReadyCondition := false for _, condition := range subnetset.Status.Conditions { - if condition.Type != nsxopv1.Ready { + if condition.Type != vpcapisv1.Ready { continue } hasReadyCondition = true @@ -101,7 +101,7 @@ func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *v // If it is, then it calls verifyNsxVpcSubnetSetStatus with the SubnetSet to verify its status. // If it's not, it returns an error. func (vp *nsxtVPCNetworkProvider) VerifyNetworkStatus(_ context.Context, clusterCtx *vmware.ClusterContext, obj runtime.Object) error { - subnetset, ok := obj.(*nsxopv1.SubnetSet) + subnetset, ok := obj.(*vpcapisv1.SubnetSet) if !ok { return fmt.Errorf("expected NSX VPC SubnetSet but got %T", obj) } @@ -124,14 +124,14 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c log.Info("Provisioning ") defer log.Info("Finished provisioning") - subnetset := &nsxopv1.SubnetSet{ + subnetset := &vpcapisv1.SubnetSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: networkNamespace, Name: networkName, }, - Spec: nsxopv1.SubnetSetSpec{ - AdvancedConfig: nsxopv1.AdvancedConfig{ - StaticIPAllocation: nsxopv1.StaticIPAllocation{ + Spec: vpcapisv1.SubnetSetSpec{ + AdvancedConfig: vpcapisv1.AdvancedConfig{ + StaticIPAllocation: vpcapisv1.StaticIPAllocation{ Enable: true, }, }, @@ -159,7 +159,7 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c // GetClusterNetworkName returns the name of a valid cluster network if one exists. func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clusterCtx *vmware.ClusterContext) (string, error) { - subnetset := &nsxopv1.SubnetSet{} + subnetset := &vpcapisv1.SubnetSet{} cluster := clusterCtx.VSphereCluster namespacedName := types.NamespacedName{ Namespace: cluster.Namespace, diff --git a/test/go.sum b/test/go.sum index 913fdb9335..a091841139 100644 --- a/test/go.sum +++ b/test/go.sum @@ -323,8 +323,8 @@ github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXV github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:cgx9UH/r53bKU/Gbv8IPsUZ34bj5+ItijA2JCUS3kVk= github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM= -github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4= -github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea h1:9BE++G23adnANH0J4UCqysUHoryCy678Yra8SCZ39FQ= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20240816055841-8081971d43ea/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU= github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU= From 8b9f04580409e6133f1600c0f3adcf21fa29c497 Mon Sep 17 00:00:00 2001 From: Yifeng Xiao Date: Mon, 26 Aug 2024 16:13:22 +0800 Subject: [PATCH 3/3] Fixup code review comments --- .golangci.yml | 7 +++-- pkg/manager/manager.go | 4 +-- pkg/services/network/constants.go | 9 +++++-- pkg/services/network/network_test.go | 32 +++++++++++------------ pkg/services/network/nsxt_vpc_provider.go | 24 +++++++---------- 5 files changed, 40 insertions(+), 36 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 9860ec3daf..f00b7751c8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -122,8 +122,11 @@ linters-settings: alias: vmwarev1 # VMware Operator - pkg: "github.com/vmware-tanzu/vm-operator/api/v1alpha2" - alias: vmoprv1 - # CABPK + alias: vmoprv1 + # NSX Operator + - pkg: "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + alias: nsxvpcv1 + # CABPK - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1 alias: bootstrapv1 # KCP diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index ae5f36f269..a5e8a22990 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -22,7 +22,7 @@ import ( "github.com/pkg/errors" netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" - vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" "gopkg.in/fsnotify.v1" @@ -65,7 +65,7 @@ func New(ctx context.Context, opts Options) (Manager, error) { _ = vmoprv1.AddToScheme(opts.Scheme) _ = ncpv1.AddToScheme(opts.Scheme) _ = netopv1.AddToScheme(opts.Scheme) - _ = vpcapisv1.AddToScheme(opts.Scheme) + _ = nsxvpcv1.AddToScheme(opts.Scheme) _ = topologyv1.AddToScheme(opts.Scheme) _ = ipamv1.AddToScheme(opts.Scheme) diff --git a/pkg/services/network/constants.go b/pkg/services/network/constants.go index 76058840b9..6e3f2faeec 100644 --- a/pkg/services/network/constants.go +++ b/pkg/services/network/constants.go @@ -19,7 +19,7 @@ package network import ( netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" - vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" ) @@ -39,6 +39,11 @@ const ( // // Deprecated: legacyDefaultNetworkLabel will be removed in a future release. legacyDefaultNetworkLabel = "capw.vmware.com/is-default-network" + + // AnnotationEnableEndpointHealthCheckKey is the key of the annotation that is used to enable health check on the + // Service endpoint port. vm-operator propagates annotations in VMService to Service and LB providers like NSX-T + // will enable health check on the endpoint target port when this annotation is present on the Service. + AnnotationEnableEndpointHealthCheckKey = "lb.iaas.vmware.com/enable-endpoint-health-check" ) var ( @@ -49,5 +54,5 @@ var ( NetworkGVKNSXT = ncpv1.SchemeGroupVersion.WithKind("VirtualNetwork") // NetworkGVKNSXTVPC is the GVK used for networks in NSX-T VPC mode. - NetworkGVKNSXTVPC = vpcapisv1.SchemeGroupVersion.WithKind("SubnetSet") + NetworkGVKNSXTVPC = nsxvpcv1.SchemeGroupVersion.WithKind("SubnetSet") ) diff --git a/pkg/services/network/network_test.go b/pkg/services/network/network_test.go index 6b26c112ef..0e2f687072 100644 --- a/pkg/services/network/network_test.go +++ b/pkg/services/network/network_test.go @@ -24,7 +24,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" - vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -270,7 +270,7 @@ var _ = Describe("Network provider", func() { Expect(vm.Spec.Network.Interfaces).To(HaveLen(1)) Expect(vm.Spec.Network.Interfaces[0].Network.Name).To(Equal(vSphereCluster.Name)) Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.Kind).To(Equal("SubnetSet")) - Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(vpcapisv1.SchemeGroupVersion.String())) + Expect(vm.Spec.Network.Interfaces[0].Network.TypeMeta.APIVersion).To(Equal(nsxvpcv1.SchemeGroupVersion.String())) }) }) }) @@ -336,7 +336,7 @@ var _ = Describe("Network provider", func() { Expect(ncpv1.AddToScheme(scheme)).To(Succeed()) Expect(corev1.AddToScheme(scheme)).To(Succeed()) Expect(vmwarev1.AddToScheme(scheme)).To(Succeed()) - Expect(vpcapisv1.AddToScheme(scheme)).To(Succeed()) + Expect(nsxvpcv1.AddToScheme(scheme)).To(Succeed()) }) Context("with dummy network provider", func() { @@ -599,14 +599,14 @@ var _ = Describe("Network provider", func() { It("should not update subnetset", func() { // Fetch the SubnetSet before the operation - initialSubnetSet := &vpcapisv1.SubnetSet{} + initialSubnetSet := &nsxvpcv1.SubnetSet{} err = client.Get(ctx, apitypes.NamespacedName{ Name: dummyCluster, Namespace: dummyNs, }, initialSubnetSet) Expect(err).NotTo(HaveOccurred()) - status := vpcapisv1.SubnetSetStatus{ - Conditions: []vpcapisv1.Condition{ + status := nsxvpcv1.SubnetSetStatus{ + Conditions: []nsxvpcv1.Condition{ { Type: "Ready", Status: "True", @@ -620,7 +620,7 @@ var _ = Describe("Network provider", func() { Expect(err).ToNot(HaveOccurred()) Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name)) - createdSubnetSet := &vpcapisv1.SubnetSet{} + createdSubnetSet := &nsxvpcv1.SubnetSet{} err = client.Get(ctx, apitypes.NamespacedName{ Name: dummyCluster, Namespace: dummyNs, @@ -633,7 +633,7 @@ var _ = Describe("Network provider", func() { It("should successfully retrieve VM service annotations, including the annotation to enable LB healthcheck", func() { annotations, err := np.GetVMServiceAnnotations(ctx, clusterCtx) Expect(err).ToNot(HaveOccurred()) - Expect(annotations).To(HaveKey("lb.iaas.vmware.com/enable-endpoint-health-check")) + Expect(annotations).To(HaveKey(AnnotationEnableEndpointHealthCheckKey)) }) }) @@ -658,7 +658,7 @@ var _ = Describe("Network provider", func() { Expect(err).ToNot(HaveOccurred()) Expect(subnetset).To(Equal(clusterCtx.VSphereCluster.Name)) - createdSubnetSet := &vpcapisv1.SubnetSet{} + createdSubnetSet := &nsxvpcv1.SubnetSet{} err = client.Get(ctx, apitypes.NamespacedName{ Name: dummyCluster, Namespace: dummyNs, @@ -700,14 +700,14 @@ var _ = Describe("Network provider", func() { BeforeEach(func() { scheme = runtime.NewScheme() - Expect(vpcapisv1.AddToScheme(scheme)).To(Succeed()) + Expect(nsxvpcv1.AddToScheme(scheme)).To(Succeed()) nsxvpcNp, _ = NSXTVpcNetworkProvider(client).(*nsxtVPCNetworkProvider) np = nsxvpcNp }) It("should return error when subnetset ready status is false", func() { - status := vpcapisv1.SubnetSetStatus{ - Conditions: []vpcapisv1.Condition{ + status := nsxvpcv1.SubnetSetStatus{ + Conditions: []nsxvpcv1.Condition{ { Type: "Ready", Status: "False", @@ -716,7 +716,7 @@ var _ = Describe("Network provider", func() { }, }, } - subnetsetObj = &vpcapisv1.SubnetSet{ + subnetsetObj = &nsxvpcv1.SubnetSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, Name: cluster.Name, @@ -732,10 +732,10 @@ var _ = Describe("Network provider", func() { }) It("should return error when subnetset ready status is not set", func() { - status := vpcapisv1.SubnetSetStatus{ - Conditions: []vpcapisv1.Condition{}, + status := nsxvpcv1.SubnetSetStatus{ + Conditions: []nsxvpcv1.Condition{}, } - subnetsetObj = &vpcapisv1.SubnetSet{ + subnetsetObj = &nsxvpcv1.SubnetSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, Name: cluster.Name, diff --git a/pkg/services/network/nsxt_vpc_provider.go b/pkg/services/network/nsxt_vpc_provider.go index f4251b9c79..6cb43503cc 100644 --- a/pkg/services/network/nsxt_vpc_provider.go +++ b/pkg/services/network/nsxt_vpc_provider.go @@ -21,7 +21,7 @@ import ( "fmt" "github.com/pkg/errors" - vpcapisv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + nsxvpcv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" vmoprv1common "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" corev1 "k8s.io/api/core/v1" @@ -40,11 +40,6 @@ import ( "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services" ) -const ( - // AnnotationEnableEndpointHealthCheckKey is the key of the annotation that is used to enable health check on the VMService endpoint port. - AnnotationEnableEndpointHealthCheckKey = "lb.iaas.vmware.com/enable-endpoint-health-check" -) - // nsxtVPCNetworkProvider provisions nsx-vpc type cluster network. type nsxtVPCNetworkProvider struct { client client.Client @@ -71,13 +66,13 @@ func (vp *nsxtVPCNetworkProvider) SupportsVMReadinessProbe() bool { // verifyNsxtVpcSubnetSetStatus checks the status conditions of a given SubnetSet within a cluster context. // If the subnet isn't ready, it is marked as false, and the function returns an error. // If the subnet is ready, the function updates the VSphereCluster with a "true" status and returns nil. -func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *vpcapisv1.SubnetSet) error { +func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *vmwarev1.VSphereCluster, subnetset *nsxvpcv1.SubnetSet) error { clusterName := vspherecluster.Name namespace := vspherecluster.Namespace hasReadyCondition := false for _, condition := range subnetset.Status.Conditions { - if condition.Type != vpcapisv1.Ready { + if condition.Type != nsxvpcv1.Ready { continue } hasReadyCondition = true @@ -101,7 +96,7 @@ func (vp *nsxtVPCNetworkProvider) verifyNsxtVpcSubnetSetStatus(vspherecluster *v // If it is, then it calls verifyNsxVpcSubnetSetStatus with the SubnetSet to verify its status. // If it's not, it returns an error. func (vp *nsxtVPCNetworkProvider) VerifyNetworkStatus(_ context.Context, clusterCtx *vmware.ClusterContext, obj runtime.Object) error { - subnetset, ok := obj.(*vpcapisv1.SubnetSet) + subnetset, ok := obj.(*nsxvpcv1.SubnetSet) if !ok { return fmt.Errorf("expected NSX VPC SubnetSet but got %T", obj) } @@ -124,14 +119,14 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c log.Info("Provisioning ") defer log.Info("Finished provisioning") - subnetset := &vpcapisv1.SubnetSet{ + subnetset := &nsxvpcv1.SubnetSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: networkNamespace, Name: networkName, }, - Spec: vpcapisv1.SubnetSetSpec{ - AdvancedConfig: vpcapisv1.AdvancedConfig{ - StaticIPAllocation: vpcapisv1.StaticIPAllocation{ + Spec: nsxvpcv1.SubnetSetSpec{ + AdvancedConfig: nsxvpcv1.AdvancedConfig{ + StaticIPAllocation: nsxvpcv1.StaticIPAllocation{ Enable: true, }, }, @@ -159,7 +154,7 @@ func (vp *nsxtVPCNetworkProvider) ProvisionClusterNetwork(ctx context.Context, c // GetClusterNetworkName returns the name of a valid cluster network if one exists. func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clusterCtx *vmware.ClusterContext) (string, error) { - subnetset := &vpcapisv1.SubnetSet{} + subnetset := &nsxvpcv1.SubnetSet{} cluster := clusterCtx.VSphereCluster namespacedName := types.NamespacedName{ Namespace: cluster.Namespace, @@ -173,6 +168,7 @@ func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clu // The GetVMServiceAnnotations method always returns an empty map representing annotations. func (vp *nsxtVPCNetworkProvider) GetVMServiceAnnotations(_ context.Context, _ *vmware.ClusterContext) (map[string]string, error) { + // The value of the annotation lb.iaas.vmware.com/enable-endpoint-health-check is expected to be an empty string. return map[string]string{AnnotationEnableEndpointHealthCheckKey: ""}, nil }