Enhance workaround for Subscription ResolutionFailed bug

https://access.redhat.com/solutions/6991414
ryanemerson · Sep 10, 2024 · c8d84c0 · c8d84c0
1 parent fe3f7f0
commit c8d84c0
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 14 deletions.
diff --git a/test/e2e/hotrod-rolling-upgrade/hotrod_rolling_upgrade_test.go b/test/e2e/hotrod-rolling-upgrade/hotrod_rolling_upgrade_test.go
@@ -64,7 +64,8 @@ func TestRollingUpgrade(t *testing.T) {
 	}
 
 	defer testKube.CleanupOLMTest(t, tutils.TestName(t), olm.SubName, olm.SubNamespace, olm.SubPackage)
-	testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub)
+	testKube.CreateOperatorGroup(olm)
+	testKube.CreateSubscriptionAndApproveInitialVersion(sub)
 
 	replicas := 2
 	entriesPerCache := 100

diff --git a/test/e2e/upgrade/dropped_operand_test.go b/test/e2e/upgrade/dropped_operand_test.go
@@ -43,7 +43,8 @@ func TestUpgradeFromDroppedOperand(t *testing.T) {
 		},
 	}
 	defer testKube.CleanupOLMTest(t, tutils.TestName(t), olm.SubName, olm.SubNamespace, olm.SubPackage)
-	testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub)
+	testKube.CreateOperatorGroup(olm)
+	testKube.CreateSubscriptionAndApproveInitialVersion(sub)
 
 	replicas := 1
 	spec := tutils.DefaultSpec(t, testKube, func(i *ispnv1.Infinispan) {

diff --git a/test/e2e/upgrade/upgrade_test.go b/test/e2e/upgrade/upgrade_test.go
@@ -63,10 +63,11 @@ func TestUpgrade(t *testing.T) {
 	}
 
 	defer testKube.CleanupOLMTest(t, tutils.TestName(t), olm.SubName, olm.SubNamespace, olm.SubPackage)
-	testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub)
+	testKube.CreateOperatorGroup(olm)
+	testKube.CreateSubscriptionAndApproveInitialVersion(sub)
 
 	// Create the Infinispan CR
-	replicas := 2
+	replicas := 1
 	spec := tutils.DefaultSpec(t, testKube, func(i *ispnv1.Infinispan) {
 		i.Spec.Replicas = int32(replicas)
 		i.Spec.Service.Container.EphemeralStorage = false

diff --git a/test/e2e/utils/olm.go b/test/e2e/utils/olm.go
@@ -94,11 +94,7 @@ func getChannel(env string, manifest *manifests.PackageManifest) manifests.Packa
 	return manifests.PackageChannel{}
 }
 
-func (k TestKubernetes) CreateSubscriptionAndApproveInitialVersion(olm OLMEnv, sub *coreos.Subscription) {
-	// Create OperatorGroup only if Subscription is created in non-global namespace
-	if olm.SubNamespace != "openshift-operators" && olm.SubNamespace != "operators" {
-		k.CreateOperatorGroup(olm.SubName, olm.SubNamespace, olm.SubNamespace)
-	}
+func (k TestKubernetes) CreateSubscriptionAndApproveInitialVersion(sub *coreos.Subscription) {
 	k.CreateSubscription(sub)
 	// Approve the initial startingCSV InstallPlan
 	k.WaitForSubscriptionState(coreos.SubscriptionStateUpgradePending, sub)
@@ -183,17 +179,22 @@ func (k TestKubernetes) CleanupOLMTest(t *testing.T, testIdentifier, subName, su
 	}
 }
 
-func (k TestKubernetes) CreateOperatorGroup(name, namespace string, targetNamespaces ...string) {
+func (k TestKubernetes) CreateOperatorGroup(olm OLMEnv) {
+	// Create OperatorGroup only if Subscription is created in non-global namespace
+	if olm.SubNamespace == "openshift-operators" || olm.SubNamespace == "operators" {
+		return
+	}
+
 	operatorGroup := &coreosv1.OperatorGroup{
 		TypeMeta: metav1.TypeMeta{
 			Kind: coreosv1.OperatorGroupKind,
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: namespace,
+			Name:      olm.SubName,
+			Namespace: olm.SubNamespace,
 		},
 		Spec: coreosv1.OperatorGroupSpec{
-			TargetNamespaces: targetNamespaces,
+			TargetNamespaces: []string{olm.SubNamespace},
 		},
 	}
 	err := k.Kubernetes.Client.Create(context.TODO(), operatorGroup)
@@ -361,7 +362,22 @@ func (k TestKubernetes) WaitForSubscription(sub *coreos.Subscription, predicate
 			// https://github.com/operator-framework/operator-lifecycle-manager/issues/2201
 			// https://issues.redhat.com/browse/OCPBUGS-5080
 			if condition.Type == "ResolutionFailed" && condition.Status == corev1.ConditionTrue {
+				fmt.Printf("CSV '%s' ResolutionFailed: %s\n", sub.Status.InstalledCSV, condition.Message)
+				fmt.Printf("Deleting Subscription '%s:%s'", sub.Name, sub.Namespace)
+				// Delete Subscription and CSV
+				k.DeleteSubscription(sub.Name, sub.Namespace)
+
+				csv := &coreos.ClusterServiceVersion{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      sub.Status.InstalledCSV,
+						Namespace: sub.Namespace,
+					},
+				}
+				fmt.Printf("Deleting CSV '%s:%s'", csv.Name, csv.Namespace)
+				ExpectMaybeNotFound(k.Kubernetes.Client.Delete(context.TODO(), csv, DeleteOpts...))
+
 				// Restart OLM pods to clear the cache
+				fmt.Println("Restarting OLM and Catalog Operator")
 				olmNamespace := k.OLMNamespace()
 				ExpectNoError(
 					k.Kubernetes.Client.DeleteAllOf(
@@ -380,6 +396,19 @@ func (k TestKubernetes) WaitForSubscription(sub *coreos.Subscription, predicate
 						client.MatchingLabels(map[string]string{"app": "catalog-operator"}),
 					),
 				)
+
+				// Create new Subscription with the startingCSV set to the failed version
+				fmt.Println("Attempting to recreate Subscription")
+				startingCSV := sub.Status.InstalledCSV
+				sub = &coreos.Subscription{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      sub.Name,
+						Namespace: sub.Namespace,
+					},
+					Spec: sub.Spec,
+				}
+				sub.Spec.StartingCSV = startingCSV
+				k.CreateSubscriptionAndApproveInitialVersion(sub)
 				err = poll()
 				break
 			}

diff --git a/test/e2e/webhook/webhook_test.go b/test/e2e/webhook/webhook_test.go
@@ -73,7 +73,8 @@ func TestMain(m *testing.M) {
 	// Defer in case a panic is encountered by one of the tests
 	defer testKube.CleanupOLMTest(nil, "webhook_test_main_defer", olm.SubName, olm.SubNamespace, olm.SubPackage)
 
-	testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub)
+	testKube.CreateOperatorGroup(olm)
+	testKube.CreateSubscriptionAndApproveInitialVersion(sub)
 	testKube.WaitForDeployment("infinispan-operator-controller-manager", olm.SubNamespace)
 
 	// We must ensure that the Operator pods have the defined environment variables available