From 64063531e858d1d56e749e3728bcb099fe9e6062 Mon Sep 17 00:00:00 2001
From: Stephen Cahill <stephen.cahill@datadoghq.com>
Date: Mon, 25 Nov 2024 18:51:06 -0500
Subject: [PATCH] seedling: Add retry to clusterctl `UpgradeWithBinary`

---
 test/framework/clusterctl/client.go           |  5 +++--
 .../clusterctl/clusterctl_helpers.go          | 21 ++++++++++++++++---
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/test/framework/clusterctl/client.go b/test/framework/clusterctl/client.go
index 95a02c993398..eab64337adb6 100644
--- a/test/framework/clusterctl/client.go
+++ b/test/framework/clusterctl/client.go
@@ -203,7 +203,7 @@ func Upgrade(ctx context.Context, input UpgradeInput) {
 }
 
 // UpgradeWithBinary calls clusterctl upgrade apply with the list of providers defined in the local repository.
-func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) {
+func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) error {
 	if len(input.ClusterctlVariables) > 0 {
 		outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName))
 		Expect(CopyAndAmendClusterctlConfig(ctx, CopyAndAmendClusterctlConfigInput{
@@ -227,8 +227,9 @@ func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) {
 		if errors.As(err, &exitErr) {
 			stdErr = string(exitErr.Stderr)
 		}
+		return fmt.Errorf("failed to run clusterctl upgrade apply:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)
 	}
-	Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade apply:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)
+	return nil
 }
 
 func calculateClusterCtlUpgradeArgs(input UpgradeInput) []string {
diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go
index 71722a96bd05..806742d5b636 100644
--- a/test/framework/clusterctl/clusterctl_helpers.go
+++ b/test/framework/clusterctl/clusterctl_helpers.go
@@ -197,12 +197,27 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen
 	client := input.ClusterProxy.GetClient()
 
 	if input.ClusterctlBinaryPath != "" {
-		UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput)
+		clusterctlVersion, err := getClusterCtlVersion(input.ClusterctlBinaryPath)
+		Expect(err).ToNot(HaveOccurred())
+		upgradeRetries := 1
+		// Older versions of clusterctl may need to retry the upgrade process to allow for
+		// cert-manager CAs to become available before continuing.  For newer versions of clusterctl
+		// this is addressed with https://github.com/kubernetes-sigs/cluster-api/pull/10513
+		if clusterctlVersion.LT(semver.MustParse("1.7.0")) {
+			upgradeRetries = 2
+		}
+		for i := range upgradeRetries {
+			err := UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput)
+			if err != nil && i < upgradeRetries-1 {
+				log.Logf("Failed to UpgradeWithBinary, retrying: %v", err)
+				continue
+			}
+			Expect(err).ToNot(HaveOccurred())
+			break
+		}
 		// Old versions of clusterctl may deploy CRDs, Mutating- and/or ValidatingWebhookConfigurations
 		// before creating the new Certificate objects. This check ensures the CA's are up to date before
 		// continuing.
-		clusterctlVersion, err := getClusterCtlVersion(input.ClusterctlBinaryPath)
-		Expect(err).ToNot(HaveOccurred())
 		if clusterctlVersion.LT(semver.MustParse("1.7.2")) {
 			Eventually(func() error {
 				return verifyCAInjection(ctx, client)