Merge pull request #523 from 14rcole/stoneintg-753

fix(stoneintg-753): requeue release creation until timeout

controllers/snapshot/snapshot_adapter.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"time"
 
 	clienterrors "k8s.io/apimachinery/pkg/api/errors"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -43,6 +44,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const RetryReleaseTimeout = time.Duration(3 * time.Hour)
+
 // configuration options for scenario
 type ScenarioOptions struct {
 	IsReRun bool
@@ -434,9 +437,9 @@ func (a *Adapter) EnsureAllReleasesExist() (controller.OperationResult, error) {
 		if er != nil {
 			a.logger.Error(er, "Failed to mark snapshot integration status as invalid",
 				"snapshot.Name", a.snapshot.Name)
-			return controller.RequeueWithError(errors.Join(err, er))
+			return a.RequeueIfYoungerThanThreshold(errors.Join(err, er))
 		}
-		return controller.RequeueWithError(err)
+		return a.RequeueIfYoungerThanThreshold(err)
 	}
 
 	err = a.createMissingReleasesForReleasePlans(a.application, releasePlans, a.snapshot)
@@ -450,9 +453,9 @@ func (a *Adapter) EnsureAllReleasesExist() (controller.OperationResult, error) {
 		if er != nil {
 			a.logger.Error(er, "Failed to mark snapshot integration status as invalid",
 				"snapshot.Name", a.snapshot.Name)
-			return controller.RequeueWithError(errors.Join(err, er))
+			return a.RequeueIfYoungerThanThreshold(errors.Join(err, er))
 		}
-		return controller.RequeueWithError(err)
+		return a.RequeueIfYoungerThanThreshold(err)
 	}
 
 	// Mark the Snapshot as already auto-released to prevent re-releasing the Snapshot when it gets reconciled
@@ -933,3 +936,17 @@ func GetDetailsFromStatusError(err error) (string, string) {
 	}
 	return errKind, errName
 }
+
+// RequeueIfYoungerThanThreshold checks if the adapter' snapshot is younger than the threshold defined
+// in the function.  If it is, the function returns an operation result instructing the reconciler
+// to requeue the object and the error message passed to the function.  If not, the function returns
+// an operation result instructing the reconciler NOT to requeue the object.
+func (a *Adapter) RequeueIfYoungerThanThreshold(retErr error) (controller.OperationResult, error) {
+	snapshotCreationTime := a.snapshot.GetCreationTimestamp().Time
+	durationSinceSnapshotCreation := time.Since(snapshotCreationTime)
+
+	if durationSinceSnapshotCreation < RetryReleaseTimeout {
+		return controller.RequeueWithError(retErr)
+	}
+	return controller.ContinueProcessing()
+}

controllers/snapshot/snapshot_adapter_test.go

@@ -995,6 +995,32 @@ var _ = Describe("Snapshot Adapter", Ordered, func() {
 			Expect(err).To(HaveOccurred())
 			Expect(buf.String()).Should(ContainSubstring("Snapshot integration status marked as Invalid. Failed to get all ReleasePlans"))
 		})
+
+		It("Returns RequeueWithError if the snapshot is less than three hours old", func() {
+			adapter = NewAdapter(hasSnapshot, hasApp, hasComp, log, loader.NewMockLoader(), k8sClient, ctx)
+			testErr := fmt.Errorf("something went wrong with the release")
+
+			result, err := adapter.RequeueIfYoungerThanThreshold(testErr)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(Equal(testErr))
+			Expect(result).NotTo(BeNil())
+			Expect(result.RequeueRequest).To(BeTrue())
+		})
+
+		It("Returns ContinueProcessing if the snapshot is greater than or equal to three hours old", func() {
+			// Set snapshot creation time to 3 hours ago
+			// time.Sub takes a time.Time and returns a time.Duration.  Time.Add takes a time.Duration
+			// and returns a time.Time.  Why?  Who knows.  We want the latter, so we add -3 hours here
+			hasSnapshot.CreationTimestamp = metav1.NewTime(time.Now().Add(-1 * RetryReleaseTimeout))
+
+			adapter = NewAdapter(hasSnapshot, hasApp, hasComp, log, loader.NewMockLoader(), k8sClient, ctx)
+			testErr := fmt.Errorf("something went wrong with the release")
+
+			result, err := adapter.RequeueIfYoungerThanThreshold(testErr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).NotTo(BeNil())
+			Expect(result.RequeueRequest).To(BeFalse())
+		})
 	})
 
 	When("multiple components exist", func() {