Exclude kube-vip

Signed-off-by: Danil Grigorev <[email protected]>
rancher · Apr 12, 2024 · 5289859 · 5289859
1 parent 8ef8e20
commit 5289859
Show file tree

Hide file tree

Showing 6 changed files with 10 additions and 208 deletions.
diff --git a/test/e2e/data/infrastructure/cluster-template-docker-legacy.yaml b/test/e2e/data/infrastructure/cluster-template-docker-legacy.yaml
@@ -75,9 +75,6 @@ spec:
       cidrBlocks:
       - 10.46.0.0/16
     serviceDomain: cluster.local
-  controlPlaneEndpoint:
-    host: "${REGISTRATION_VIP}"
-    port: 6443
   controlPlaneRef:
     apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
     kind: RKE2ControlPlane
@@ -115,46 +112,6 @@ spec:
     kind: DockerMachineTemplate
     name:  "${CLUSTER_NAME}-control-plane"
   nodeDrainTimeout: 30s
-  registrationMethod: "address"
-  registrationAddress: "${REGISTRATION_VIP}"
-  preRKE2Commands:
-  - mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
-  files:
-  - path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
-    content: |
-      apiVersion: v1
-      kind: ServiceAccount
-      metadata:
-        name: kube-vip
-        namespace: kube-system
-      ---
-      apiVersion: rbac.authorization.k8s.io/v1
-      kind: ClusterRole
-      metadata:
-        annotations:
-          rbac.authorization.kubernetes.io/autoupdate: "true"
-        name: system:kube-vip-role
-      rules:
-        - apiGroups: [""]
-          resources: ["services", "services/status", "nodes", "endpoints"]
-          verbs: ["list","get","watch", "update"]
-        - apiGroups: ["coordination.k8s.io"]
-          resources: ["leases"]
-          verbs: ["list", "get", "watch", "update", "create"]
-      ---
-      kind: ClusterRoleBinding
-      apiVersion: rbac.authorization.k8s.io/v1
-      metadata:
-        name: system:kube-vip-binding
-      roleRef:
-        apiGroup: rbac.authorization.k8s.io
-        kind: ClusterRole
-        name: system:kube-vip-role
-      subjects:
-      - kind: ServiceAccount
-        name: kube-vip
-        namespace: kube-system
-    owner: root:root
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: DockerMachineTemplate

diff --git a/test/e2e/data/infrastructure/cluster-template-docker.yaml b/test/e2e/data/infrastructure/cluster-template-docker.yaml
@@ -75,9 +75,6 @@ spec:
       cidrBlocks:
       - 10.46.0.0/16
     serviceDomain: cluster.local
-  controlPlaneEndpoint:
-    host: "${REGISTRATION_VIP}"
-    port: 6443
   controlPlaneRef:
     apiVersion: controlplane.cluster.x-k8s.io/v1beta1
     kind: RKE2ControlPlane
@@ -115,46 +112,6 @@ spec:
     kind: DockerMachineTemplate
     name:  "${CLUSTER_NAME}-control-plane"
   nodeDrainTimeout: 30s
-  registrationMethod: "address"
-  registrationAddress: "${REGISTRATION_VIP}"
-  preRKE2Commands:
-  - mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
-  files:
-  - path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
-    content: |
-      apiVersion: v1
-      kind: ServiceAccount
-      metadata:
-        name: kube-vip
-        namespace: kube-system
-      ---
-      apiVersion: rbac.authorization.k8s.io/v1
-      kind: ClusterRole
-      metadata:
-        annotations:
-          rbac.authorization.kubernetes.io/autoupdate: "true"
-        name: system:kube-vip-role
-      rules:
-        - apiGroups: [""]
-          resources: ["services", "services/status", "nodes", "endpoints"]
-          verbs: ["list","get","watch", "update"]
-        - apiGroups: ["coordination.k8s.io"]
-          resources: ["leases"]
-          verbs: ["list", "get", "watch", "update", "create"]
-      ---
-      kind: ClusterRoleBinding
-      apiVersion: rbac.authorization.k8s.io/v1
-      metadata:
-        name: system:kube-vip-binding
-      roleRef:
-        apiGroup: rbac.authorization.k8s.io
-        kind: ClusterRole
-        name: system:kube-vip-role
-      subjects:
-      - kind: ServiceAccount
-        name: kube-vip
-        namespace: kube-system
-    owner: root:root
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: DockerMachineTemplate

diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -44,7 +44,6 @@ var _ = Describe("Workload cluster creation", func() {
 		result              *ApplyClusterTemplateAndWaitResult
 		clusterName         string
 		clusterctlLogFolder string
-		registrationIP      string
 	)
 
 	BeforeEach(func() {
@@ -55,8 +54,6 @@ var _ = Describe("Workload cluster creation", func() {
 
 		Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
 
-		registrationIP = randomIp()
-
 		By("Initializing the bootstrap cluster")
 		initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder)
 
@@ -106,9 +103,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(1)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -130,9 +124,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(3)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -153,9 +144,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(3)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -189,9 +177,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
 					ControlPlaneMachineCount: ptr.To(int64(3)),
 					WorkerMachineCount:       ptr.To(int64(3)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -218,9 +203,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(3)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -247,9 +229,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(1)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),

diff --git a/test/e2e/e2e_upgrade_test.go b/test/e2e/e2e_upgrade_test.go
@@ -44,7 +44,6 @@ var _ = Describe("Workload cluster creation", func() {
 		result              *ApplyClusterTemplateAndWaitResult
 		clusterName         string
 		clusterctlLogFolder string
-		registrationIP      string
 	)
 
 	BeforeEach(func() {
@@ -55,8 +54,6 @@ var _ = Describe("Workload cluster creation", func() {
 
 		Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
 
-		registrationIP = randomIp()
-
 		clusterName = fmt.Sprintf("caprke2-e2e-%s-upgrade", util.RandomString(6))
 
 		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
@@ -107,9 +104,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
 					ControlPlaneMachineCount: ptr.To(int64(3)),
 					WorkerMachineCount:       ptr.To(int64(1)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -146,9 +140,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
 					ControlPlaneMachineCount: ptr.To(int64(2)),
 					WorkerMachineCount:       ptr.To(int64(1)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -177,9 +168,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
 					ControlPlaneMachineCount: ptr.To(int64(2)),
 					WorkerMachineCount:       ptr.To(int64(1)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -212,9 +200,6 @@ var _ = Describe("Workload cluster creation", func() {
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(1)),
-					ClusterctlVariables: map[string]string{
-						"REGISTRATION_VIP": registrationIP,
-					},
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
 				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),

diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go
@@ -21,20 +21,15 @@ package e2e
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
-	"math/rand"
-	"net"
 	"os/exec"
-	"sort"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/apps/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 
@@ -164,58 +159,27 @@ func DiscoveryAndWaitForMachineDeployments(ctx context.Context, input framework.
 		ClusterName: input.Cluster.Name,
 		Namespace:   input.Cluster.Namespace,
 	})
-	for _, deployment := range machineDeployments {
-		WaitForMachineDeploymentNodesToExist(ctx, framework.WaitForMachineDeploymentNodesToExistInput{
-			Lister:            input.Lister,
-			Cluster:           input.Cluster,
-			MachineDeployment: deployment,
-		}, intervals...)
 
+	for _, deployment := range machineDeployments {
 		framework.AssertMachineDeploymentFailureDomains(ctx, framework.AssertMachineDeploymentFailureDomainsInput{
 			Lister:            input.Lister,
 			Cluster:           input.Cluster,
 			MachineDeployment: deployment,
 		})
 	}
-	return machineDeployments
-}
-
-// WaitForMachineDeploymentNodesToExist waits until all nodes associated with a machine deployment exist.
-func WaitForMachineDeploymentNodesToExist(ctx context.Context, input framework.WaitForMachineDeploymentNodesToExistInput, intervals ...interface{}) {
-	Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachineDeploymentNodesToExist")
-	Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling WaitForMachineDeploymentNodesToExist")
-	Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentNodesToExist")
 
-	By("Waiting for the workload nodes to exist")
 	Eventually(func(g Gomega) {
-		selectorMap, err := metav1.LabelSelectorAsMap(&input.MachineDeployment.Spec.Selector)
-		g.Expect(err).ToNot(HaveOccurred())
-		ms := &clusterv1.MachineSetList{}
-		err = input.Lister.List(ctx, ms, client.InNamespace(input.Cluster.Namespace), client.MatchingLabels(selectorMap))
-		g.Expect(err).ToNot(HaveOccurred())
-		g.Expect(ms.Items).NotTo(BeEmpty())
-		machineSet := ms.Items[0]
-		sort.Slice(ms.Items, func(i, j int) bool {
-			return ms.Items[j].CreationTimestamp.After(ms.Items[i].CreationTimestamp.Time)
+		machineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
+			Lister:      input.Lister,
+			ClusterName: input.Cluster.Name,
+			Namespace:   input.Cluster.Namespace,
 		})
-		for _, ms := range ms.Items {
-			if *machineSet.Spec.Replicas == *input.MachineDeployment.Spec.Replicas {
-				machineSet = ms
-			}
+		for _, deployment := range machineDeployments {
+			g.Expect(*deployment.Spec.Replicas).To(BeEquivalentTo(deployment.Status.ReadyReplicas))
 		}
-		selectorMap, err = metav1.LabelSelectorAsMap(&machineSet.Spec.Selector)
-		g.Expect(err).ToNot(HaveOccurred())
-		machines := &clusterv1.MachineList{}
-		err = input.Lister.List(ctx, machines, client.InNamespace(machineSet.Namespace), client.MatchingLabels(selectorMap))
-		g.Expect(err).ToNot(HaveOccurred())
-		count := 0
-		for _, machine := range machines.Items {
-			if machine.Status.NodeRef != nil {
-				count++
-			}
-		}
-		g.Expect(count).To(Equal(int(*input.MachineDeployment.Spec.Replicas)))
-	}, intervals...).Should(Succeed(), "Timed out waiting for %d nodes to be created for MachineDeployment %s", int(*input.MachineDeployment.Spec.Replicas), klog.KObj(input.MachineDeployment))
+	}, intervals...).Should(Succeed())
+
+	return machineDeployments
 }
 
 func SetControllerVersionAndWait(ctx context.Context, proxy framework.ClusterProxy, version string) {
@@ -526,37 +490,3 @@ func CollectArtifacts(ctx context.Context, kubeconfigPath, name string, args ...
 	fmt.Printf("stdout:\n%s\n", string(stdout.Bytes()))
 	return err
 }
-
-type networks []network
-
-type network struct {
-	IPAM ipaminfo `json:"IPAM"`
-}
-
-type ipaminfo struct {
-	Config []ipamnetworkconfig `json:"Config"`
-}
-
-type ipamnetworkconfig struct {
-	Subnet string `json:"Subnet"`
-}
-
-func randRange(min, max int) int {
-	return rand.Intn(max-min) + min
-}
-
-func randomIp() string {
-	data, err := exec.Command("docker", "network", "inspect", "kind").CombinedOutput()
-	Expect(err).ToNot(HaveOccurred())
-
-	networks := networks{}
-	err = json.Unmarshal(data, &networks)
-	Expect(err).ToNot(HaveOccurred())
-
-	ip, ipnet, err := net.ParseCIDR(networks[0].IPAM.Config[0].Subnet)
-	Expect(err).ToNot(HaveOccurred())
-	ip = ip.Mask(ipnet.Mask)
-	ip[3] += byte(randRange(25, 254))
-
-	return ip.String()
-}