From c8d84c010b5f0f6f3ba8244397dfee33e29d69f7 Mon Sep 17 00:00:00 2001 From: Ryan Emerson Date: Tue, 10 Sep 2024 11:51:13 +0100 Subject: [PATCH] Enhance workaround for Subscription ResolutionFailed bug https://access.redhat.com/solutions/6991414 --- .../hotrod_rolling_upgrade_test.go | 3 +- test/e2e/upgrade/dropped_operand_test.go | 3 +- test/e2e/upgrade/upgrade_test.go | 5 +- test/e2e/utils/olm.go | 47 +++++++++++++++---- test/e2e/webhook/webhook_test.go | 3 +- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/test/e2e/hotrod-rolling-upgrade/hotrod_rolling_upgrade_test.go b/test/e2e/hotrod-rolling-upgrade/hotrod_rolling_upgrade_test.go index 9a13b55f5..a17b5bbad 100644 --- a/test/e2e/hotrod-rolling-upgrade/hotrod_rolling_upgrade_test.go +++ b/test/e2e/hotrod-rolling-upgrade/hotrod_rolling_upgrade_test.go @@ -64,7 +64,8 @@ func TestRollingUpgrade(t *testing.T) { } defer testKube.CleanupOLMTest(t, tutils.TestName(t), olm.SubName, olm.SubNamespace, olm.SubPackage) - testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub) + testKube.CreateOperatorGroup(olm) + testKube.CreateSubscriptionAndApproveInitialVersion(sub) replicas := 2 entriesPerCache := 100 diff --git a/test/e2e/upgrade/dropped_operand_test.go b/test/e2e/upgrade/dropped_operand_test.go index a825ea4dc..9a6782587 100644 --- a/test/e2e/upgrade/dropped_operand_test.go +++ b/test/e2e/upgrade/dropped_operand_test.go @@ -43,7 +43,8 @@ func TestUpgradeFromDroppedOperand(t *testing.T) { }, } defer testKube.CleanupOLMTest(t, tutils.TestName(t), olm.SubName, olm.SubNamespace, olm.SubPackage) - testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub) + testKube.CreateOperatorGroup(olm) + testKube.CreateSubscriptionAndApproveInitialVersion(sub) replicas := 1 spec := tutils.DefaultSpec(t, testKube, func(i *ispnv1.Infinispan) { diff --git a/test/e2e/upgrade/upgrade_test.go b/test/e2e/upgrade/upgrade_test.go index 24715f400..4ef99a438 100644 --- a/test/e2e/upgrade/upgrade_test.go +++ b/test/e2e/upgrade/upgrade_test.go @@ -63,10 +63,11 @@ func TestUpgrade(t *testing.T) { } defer testKube.CleanupOLMTest(t, tutils.TestName(t), olm.SubName, olm.SubNamespace, olm.SubPackage) - testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub) + testKube.CreateOperatorGroup(olm) + testKube.CreateSubscriptionAndApproveInitialVersion(sub) // Create the Infinispan CR - replicas := 2 + replicas := 1 spec := tutils.DefaultSpec(t, testKube, func(i *ispnv1.Infinispan) { i.Spec.Replicas = int32(replicas) i.Spec.Service.Container.EphemeralStorage = false diff --git a/test/e2e/utils/olm.go b/test/e2e/utils/olm.go index c807aa0f9..7c0a7db92 100644 --- a/test/e2e/utils/olm.go +++ b/test/e2e/utils/olm.go @@ -94,11 +94,7 @@ func getChannel(env string, manifest *manifests.PackageManifest) manifests.Packa return manifests.PackageChannel{} } -func (k TestKubernetes) CreateSubscriptionAndApproveInitialVersion(olm OLMEnv, sub *coreos.Subscription) { - // Create OperatorGroup only if Subscription is created in non-global namespace - if olm.SubNamespace != "openshift-operators" && olm.SubNamespace != "operators" { - k.CreateOperatorGroup(olm.SubName, olm.SubNamespace, olm.SubNamespace) - } +func (k TestKubernetes) CreateSubscriptionAndApproveInitialVersion(sub *coreos.Subscription) { k.CreateSubscription(sub) // Approve the initial startingCSV InstallPlan k.WaitForSubscriptionState(coreos.SubscriptionStateUpgradePending, sub) @@ -183,17 +179,22 @@ func (k TestKubernetes) CleanupOLMTest(t *testing.T, testIdentifier, subName, su } } -func (k TestKubernetes) CreateOperatorGroup(name, namespace string, targetNamespaces ...string) { +func (k TestKubernetes) CreateOperatorGroup(olm OLMEnv) { + // Create OperatorGroup only if Subscription is created in non-global namespace + if olm.SubNamespace == "openshift-operators" || olm.SubNamespace == "operators" { + return + } + operatorGroup := &coreosv1.OperatorGroup{ TypeMeta: metav1.TypeMeta{ Kind: coreosv1.OperatorGroupKind, }, ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, + Name: olm.SubName, + Namespace: olm.SubNamespace, }, Spec: coreosv1.OperatorGroupSpec{ - TargetNamespaces: targetNamespaces, + TargetNamespaces: []string{olm.SubNamespace}, }, } err := k.Kubernetes.Client.Create(context.TODO(), operatorGroup) @@ -361,7 +362,22 @@ func (k TestKubernetes) WaitForSubscription(sub *coreos.Subscription, predicate // https://github.com/operator-framework/operator-lifecycle-manager/issues/2201 // https://issues.redhat.com/browse/OCPBUGS-5080 if condition.Type == "ResolutionFailed" && condition.Status == corev1.ConditionTrue { + fmt.Printf("CSV '%s' ResolutionFailed: %s\n", sub.Status.InstalledCSV, condition.Message) + fmt.Printf("Deleting Subscription '%s:%s'", sub.Name, sub.Namespace) + // Delete Subscription and CSV + k.DeleteSubscription(sub.Name, sub.Namespace) + + csv := &coreos.ClusterServiceVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: sub.Status.InstalledCSV, + Namespace: sub.Namespace, + }, + } + fmt.Printf("Deleting CSV '%s:%s'", csv.Name, csv.Namespace) + ExpectMaybeNotFound(k.Kubernetes.Client.Delete(context.TODO(), csv, DeleteOpts...)) + // Restart OLM pods to clear the cache + fmt.Println("Restarting OLM and Catalog Operator") olmNamespace := k.OLMNamespace() ExpectNoError( k.Kubernetes.Client.DeleteAllOf( @@ -380,6 +396,19 @@ func (k TestKubernetes) WaitForSubscription(sub *coreos.Subscription, predicate client.MatchingLabels(map[string]string{"app": "catalog-operator"}), ), ) + + // Create new Subscription with the startingCSV set to the failed version + fmt.Println("Attempting to recreate Subscription") + startingCSV := sub.Status.InstalledCSV + sub = &coreos.Subscription{ + ObjectMeta: metav1.ObjectMeta{ + Name: sub.Name, + Namespace: sub.Namespace, + }, + Spec: sub.Spec, + } + sub.Spec.StartingCSV = startingCSV + k.CreateSubscriptionAndApproveInitialVersion(sub) err = poll() break } diff --git a/test/e2e/webhook/webhook_test.go b/test/e2e/webhook/webhook_test.go index 5417f6358..827fdc583 100644 --- a/test/e2e/webhook/webhook_test.go +++ b/test/e2e/webhook/webhook_test.go @@ -73,7 +73,8 @@ func TestMain(m *testing.M) { // Defer in case a panic is encountered by one of the tests defer testKube.CleanupOLMTest(nil, "webhook_test_main_defer", olm.SubName, olm.SubNamespace, olm.SubPackage) - testKube.CreateSubscriptionAndApproveInitialVersion(olm, sub) + testKube.CreateOperatorGroup(olm) + testKube.CreateSubscriptionAndApproveInitialVersion(sub) testKube.WaitForDeployment("infinispan-operator-controller-manager", olm.SubNamespace) // We must ensure that the Operator pods have the defined environment variables available