Add Nutanix CCM ignore node IPs list (#9068)

* BUGFIX Add Add Nutanix CCM ignore node IPs list

* Fix comments and linter errors
 - fix kube-vip validation
 - fix kube-vip add to ignore node list
 - add unit-test

* Fix comments

config/crd/bases/anywhere.eks.amazonaws.com_nutanixdatacenterconfigs.yaml

@@ -42,6 +42,13 @@ spec:
                   bundle for users that configured their Prism Central with certificates
                   from non-publicly trusted CAs
                 type: string
+              ccmExcludeNodeIPs:
+                description: CcmExcludeIPs is the optional list of IP addresses that
+                  should be excluded from the CCM IP pool for nodes. List should be
+                  valid IP addresses and IP address ranges.
+                items:
+                  type: string
+                type: array
               credentialRef:
                 description: CredentialRef is the reference to the secret name that
                   contains the credentials for the Nutanix Prism Central. The namespace

pkg/api/v1alpha1/nutanixdatacenterconfig_types.go

@@ -47,6 +47,11 @@ type NutanixDatacenterConfigSpec struct {
 	// FailureDomains is the optional list of failure domains for the Nutanix Datacenter.
 	// +optional
 	FailureDomains []NutanixDatacenterFailureDomain `json:"failureDomains,omitempty"`
+
+	// CcmExcludeIPs is the optional list of IP addresses that should be excluded from the CCM IP pool for nodes.
+	// List should be valid IP addresses and IP address ranges.
+	// +optional
+	CcmExcludeNodeIPs []string `json:"ccmExcludeNodeIPs,omitempty"`
 }
 
 // NutanixDatacenterFailureDomain defines the failure domain for the Nutanix Datacenter.

pkg/providers/nutanix/config/cp-template.yaml

@@ -647,7 +647,8 @@ data:
           "enableCustomLabeling": false,
           "topologyDiscovery": {
             "type": "Prism"
-          }
+          },
+          "ignoredNodeIPs": [{{ range $i, $ip := .ccmIgnoredNodeIPs }}{{ if $i }}, {{ end }}"{{ $ip }}"{{ end }}]
         }
     ---
     apiVersion: rbac.authorization.k8s.io/v1

pkg/providers/nutanix/template.go

@@ -4,6 +4,8 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"net"
+	"strings"
 
 	"sigs.k8s.io/yaml"
 
@@ -176,9 +178,12 @@ func buildTemplateMapCP(
 
 	failureDomains := generateNutanixFailureDomains(datacenterSpec.FailureDomains)
 
+	ccmIgnoredNodeIPs := generateCcmIgnoredNodeIPsList(clusterSpec)
+
 	values := map[string]interface{}{
 		"auditPolicy":                  auditPolicy,
 		"apiServerExtraArgs":           apiServerExtraArgs.ToPartialYaml(),
+		"ccmIgnoredNodeIPs":            ccmIgnoredNodeIPs,
 		"cloudProviderImage":           versionsBundle.Nutanix.CloudProvider.VersionedImage(),
 		"clusterName":                  clusterSpec.Cluster.Name,
 		"controlPlaneEndpointIp":       clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host,
@@ -568,3 +573,89 @@ func generateNutanixFailureDomains(eksNutanixFailureDomains []v1alpha1.NutanixDa
 	}
 	return failureDomains
 }
+
+func incrementIP(ip net.IP) {
+	for i := len(ip) - 1; i >= 0; i-- {
+		ip[i]++
+		if ip[i] > 0 {
+			break
+		}
+	}
+}
+
+func compareIP(ip1, ip2 net.IP) (int, error) {
+	if len(ip1) != len(ip2) {
+		return -1, fmt.Errorf("IP addresses are not the same protocol")
+	}
+
+	for i := 0; i < len(ip1); i++ {
+		if ip1[i] < ip2[i] {
+			return -1, nil
+		}
+		if ip1[i] > ip2[i] {
+			return 1, nil
+		}
+	}
+
+	return 0, nil
+}
+
+func addCIDRToIgnoredNodeIPsList(cidr string, result []string) []string {
+	ip, ipNet, _ := net.ParseCIDR(cidr)
+
+	// Add all ip addresses in the range to the list
+	for ip := ip.Mask(ipNet.Mask); ipNet.Contains(ip); incrementIP(ip) {
+		if ip != nil {
+			result = append(result, ip.String())
+		}
+	}
+
+	return result
+}
+
+func addIPRangeToIgnoredNodeIPsList(ipRangeStr string, result []string) []string {
+	// Parse the range
+	ipRange := strings.Split(ipRangeStr, "-")
+
+	// Parse the start and end of the range
+	start := net.ParseIP(strings.TrimSpace(ipRange[0]))
+	end := net.ParseIP(strings.TrimSpace(ipRange[1]))
+
+	cmp, _ := compareIP(start, end)
+	if cmp >= 0 {
+		// swap start and end if start is greater than end
+		start, end = end, start
+	}
+
+	// Add all ip addresses in the range to the list
+	for ip := start; !ip.Equal(end); incrementIP(ip) {
+		result = append(result, ip.String())
+	}
+
+	result = append(result, end.String())
+
+	return result
+}
+
+func addIPAddressToIgnoredNodeIPsList(ipAddrStr string, result []string) []string {
+	result = append(result, ipAddrStr)
+	return result
+}
+
+func generateCcmIgnoredNodeIPsList(clusterSpec *cluster.Spec) []string {
+	// Add the kube-vip IP address to the list
+	result := []string{clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host}
+
+	for _, IPAddrOrRange := range clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs {
+		addrOrRange := strings.TrimSpace(IPAddrOrRange)
+		if strings.Contains(addrOrRange, "/") {
+			result = addCIDRToIgnoredNodeIPsList(addrOrRange, result)
+		} else if strings.Contains(addrOrRange, "-") {
+			result = addIPRangeToIgnoredNodeIPsList(addrOrRange, result)
+		} else {
+			result = addIPAddressToIgnoredNodeIPsList(addrOrRange, result)
+		}
+	}
+
+	return result
+}

pkg/providers/nutanix/template_test.go

@@ -848,6 +848,47 @@ func TestTemplateBuilderGPUs(t *testing.T) {
 	}
 }
 
+func TestTemplateBuilderCcmExcludeNodeIPs(t *testing.T) {
+	for _, tc := range []struct {
+		Input    string
+		Output   string
+		ChangeFn func(clusterSpec *cluster.Spec) *cluster.Spec
+	}{
+		{
+			Input:  "testdata/eksa-cluster-ccm-exclude-node-ips.yaml",
+			Output: "testdata/expected_cluster_ccm_exclude_node_ips.yaml",
+			ChangeFn: func(clusterSpec *cluster.Spec) *cluster.Spec {
+				excludeNodeIPs := []string{
+					"127.100.200.101",
+					"10.10.10.10-10.10.10.13",
+					"10.123.0.0/29",
+				}
+				clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs = excludeNodeIPs
+
+				return clusterSpec
+			},
+		},
+	} {
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+
+		machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
+		creds := GetCredsFromEnv()
+
+		clusterSpec = tc.ChangeFn(clusterSpec)
+
+		bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,
+			map[string]anywherev1.NutanixMachineConfigSpec{}, creds, time.Now)
+
+		cpSpec, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
+		assert.NoError(t, err)
+		assert.NotNil(t, cpSpec)
+		test.AssertContentToFile(t, string(cpSpec), tc.Output)
+	}
+}
+
 func minimalNutanixConfigSpec(t *testing.T) (*anywherev1.NutanixDatacenterConfig, *anywherev1.NutanixMachineConfig, map[string]anywherev1.NutanixMachineConfigSpec) {
 	dcConf := &anywherev1.NutanixDatacenterConfig{}
 	err := yaml.Unmarshal([]byte(nutanixDatacenterConfigSpec), dcConf)

pkg/providers/nutanix/testdata/cluster_api_server_cert_san_domain_name.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     certSANs: ["foo.bar"]
     machineGroupRef:
       name: test

pkg/providers/nutanix/testdata/cluster_api_server_cert_san_ip.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     certSANs: ["11.11.11.11"]
     machineGroupRef:
       name: test

pkg/providers/nutanix/testdata/cluster_nutanix_etcd_encryption.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_etcd_encryption_1_29.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_failure_domains.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_invalid_trust_bundle.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_trust_bundle.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_upgrade_strategy_cp.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_upgrade_strategy_md.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips.yaml

@@ -0,0 +1,15 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_cidr.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 10.100.0.0//16

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 244.244.01

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip_range1.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 10.100.0.10-10.100.10.10-10.100.20.30

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip_range2.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 192.179.1.1-10.1.1.1

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip_range3.yaml

@@ -0,0 +1,17 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 10.0.10.0-::1
+

pkg/providers/nutanix/testdata/eksa-cluster-additional-categories.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig