Merge pull request #134 from markusewalker/markusv29/fix-release-issues

[v2.9] Fix flaky nodescaling test cases
rancher · Apr 8, 2024 · 3346099 · 3346099
2 parents d27fed3 + 0882b08
commit 3346099
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 14 deletions.
diff --git a/extensions/machinepools/machinepools.go b/extensions/machinepools/machinepools.go
@@ -1,6 +1,7 @@
 package machinepools
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 	"strings"
@@ -72,14 +73,19 @@ func updateMachinePoolQuantity(client *rancher.Client, cluster *v1.SteveAPIObjec
 		return nil, err
 	}
 
-	err = kwait.Poll(500*time.Millisecond, defaults.TenMinuteTimeout, func() (done bool, err error) {
+	err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.ThirtyMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
+		client, err = client.ReLogin()
+		if err != nil {
+			return false, err
+		}
+
 		clusterResp, err := client.Steve.SteveType("provisioning.cattle.io.cluster").ByID(cluster.ID)
 		if err != nil {
 			return false, err
 		}
 
 		if clusterResp.ObjectMeta.State.Name == active &&
-			nodestat.AllManagementNodeReady(client, cluster.ID, defaults.ThirtyMinuteTimeout) == nil {
+			nodestat.AllMachineReady(client, cluster.ID, defaults.ThirtyMinuteTimeout) == nil {
 			return true, nil
 		}
 

diff --git a/extensions/provisioning/creates.go b/extensions/provisioning/creates.go
@@ -44,9 +44,10 @@ import (
 )
 
 const (
-	active     = "active"
-	internalIP = "rke2.io/internal-ip"
-	namespace  = "fleet-default"
+	active         = "active"
+	internalIP     = "alpha.kubernetes.io/provided-node-ip"
+	rke1ExternalIP = "rke.cattle.io/external-ip"
+	namespace      = "fleet-default"
 
 	rke2k3sAirgapCustomCluster           = "rke2k3sairgapcustomcluster"
 	rke2k3sNodeCorralName                = "rke2k3sregisterNode"
@@ -805,13 +806,14 @@ func AddRKE2K3SCustomClusterNodes(client *rancher.Client, cluster *v1.SteveAPIOb
 		logrus.Infof(output)
 	}
 
-	err = kwait.Poll(500*time.Millisecond, defaults.TenMinuteTimeout, func() (done bool, err error) {
+	err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.ThirtyMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
 		clusterResp, err := client.Steve.SteveType(clusters.ProvisioningSteveResourceType).ByID(cluster.ID)
 		if err != nil {
 			return false, err
 		}
 
-		if clusterResp.ObjectMeta.State.Name == active && nodestat.AllManagementNodeReady(client, cluster.ID, defaults.ThirtyMinuteTimeout) == nil {
+		if clusterResp.ObjectMeta.State.Name == active &&
+			nodestat.AllMachineReady(client, cluster.ID, defaults.ThirtyMinuteTimeout) == nil {
 			return true, nil
 		}
 		return false, nil
@@ -849,7 +851,7 @@ func DeleteRKE2K3SCustomClusterNodes(client *rancher.Client, clusterID string, c
 					return err
 				}
 
-				err = kwait.Poll(500*time.Millisecond, defaults.TenMinuteTimeout, func() (done bool, err error) {
+				err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.ThirtyMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
 					_, err = client.Steve.SteveType(machineSteveResourceType).ByID(machine.ID)
 					if err != nil {
 						logrus.Infof("Node has successfully been deleted!")
@@ -887,13 +889,19 @@ func AddRKE1CustomClusterNodes(client *rancher.Client, cluster *management.Clust
 		logrus.Infof(output)
 	}
 
-	err = kwait.Poll(500*time.Millisecond, defaults.TenMinuteTimeout, func() (done bool, err error) {
+	err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.ThirtyMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
+		client, err = client.ReLogin()
+		if err != nil {
+			return false, err
+		}
+
 		clusterResp, err := client.Management.Cluster.ByID(cluster.ID)
 		if err != nil {
 			return false, err
 		}
 
-		if clusterResp.State == active && nodestat.AllManagementNodeReady(client, cluster.ID, defaults.ThirtyMinuteTimeout) == nil {
+		if clusterResp.State == active &&
+			nodestat.AllManagementNodeReady(client, cluster.ID, defaults.ThirtyMinuteTimeout) == nil {
 			return true, nil
 		}
 		return false, nil
@@ -916,7 +924,7 @@ func DeleteRKE1CustomClusterNodes(client *rancher.Client, cluster *management.Cl
 
 	for _, nodeToDelete := range nodesToDelete {
 		for _, node := range nodes.Data {
-			if node.ExternalIPAddress == nodeToDelete.PublicIPAddress {
+			if node.Annotations[rke1ExternalIP] == nodeToDelete.PublicIPAddress {
 				machine, err := client.Management.Node.ByID(node.ID)
 				if err != nil {
 					return err
@@ -928,7 +936,7 @@ func DeleteRKE1CustomClusterNodes(client *rancher.Client, cluster *management.Cl
 					return err
 				}
 
-				err = kwait.Poll(500*time.Millisecond, defaults.TenMinuteTimeout, func() (done bool, err error) {
+				err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.ThirtyMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
 					_, err = client.Management.Node.ByID(machine.ID)
 					if err != nil {
 						logrus.Infof("Node has successfully been deleted!")

diff --git a/extensions/rke1/nodepools/nodepools.go b/extensions/rke1/nodepools/nodepools.go
@@ -1,6 +1,7 @@
 package rke1
 
 import (
+	"context"
 	"strconv"
 	"time"
 
@@ -110,14 +111,13 @@ func updateNodePoolQuantity(client *rancher.Client, cluster *management.Cluster,
 		return nil, err
 	}
 
-	err = kwait.Poll(500*time.Millisecond, defaults.TenMinuteTimeout, func() (done bool, err error) {
+	err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.ThirtyMinuteTimeout, true, func(ctx context.Context) (done bool, err error) {
 		clusterResp, err := client.Management.Cluster.ByID(cluster.ID)
 		if err != nil {
 			return false, err
 		}
 
 		if clusterResp.State == active && nodestat.AllManagementNodeReady(client, clusterResp.ID, defaults.ThirtyMinuteTimeout) == nil {
-			logrus.Infof("Node pool is scaled!")
 			return true, nil
 		}
 		return false, nil