From def4b61e33f1b6ab6543381ef8c1af99cdf343d6 Mon Sep 17 00:00:00 2001 From: Kashif Khan Date: Tue, 24 Jan 2023 13:04:42 +0200 Subject: [PATCH] Remove and reinstall BMO in source cluster before and after pivot --- hack/tools/go.mod | 2 + hack/tools/go.sum | 6 +++ test/e2e/pivoting.go | 58 ++++++++++++++++++--- test/e2e/upgrade_management_cluster_test.go | 10 ++-- 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/hack/tools/go.mod b/hack/tools/go.mod index 6044de7dd3..8b4d22d6ab 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -53,6 +53,7 @@ require ( github.com/go-critic/go-critic v0.6.3 // indirect github.com/go-errors/errors v1.0.1 // indirect github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/zapr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.19.14 // indirect @@ -208,6 +209,7 @@ require ( mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect mvdan.cc/unparam v0.0.0-20220706161116-678bad134442 // indirect + sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20230123221616-613648eda783 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/kustomize/api v0.11.4 // indirect sigs.k8s.io/kustomize/cmd/config v0.10.6 // indirect diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 7ebe1950e0..e1a19e17be 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -214,6 +214,8 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -865,6 +867,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -876,6 +879,7 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1414,6 +1418,8 @@ mvdan.cc/unparam v0.0.0-20220706161116-678bad134442/go.mod h1:F/Cxw/6mVrNKqrR2Yj rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20230123221616-613648eda783 h1:prp6vgzsBMYx7WPI4uxvK207oaERygj3ZYeXKwXhM58= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20230123221616-613648eda783/go.mod h1:Lm5xRgQejdMHAz81exSpqvwEkIdTfoNtUDA6MM4kltw= sigs.k8s.io/controller-tools v0.10.0 h1:0L5DTDTFB67jm9DkfrONgTGmfc/zYow0ZaHyppizU2U= sigs.k8s.io/controller-tools v0.10.0/go.mod h1:uvr0EW6IsprfB0jpQq6evtKy+hHyHCXNfdWI5ONPx94= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= diff --git a/test/e2e/pivoting.go b/test/e2e/pivoting.go index 79b11a5d3a..0d53ac3aaa 100644 --- a/test/e2e/pivoting.go +++ b/test/e2e/pivoting.go @@ -33,6 +33,7 @@ const ( ironicBasicAuth = "IRONIC_BASIC_AUTH" Kind = "kind" NamePrefix = "NAMEPREFIX" + NameSuffix = "NAMESUFFIX" restartContainerCertUpdate = "RESTART_CONTAINER_CERTIFICATE_UPDATED" ironicNamespace = "IRONIC_NAMESPACE" ) @@ -86,12 +87,13 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) { if ephemeralCluster == Kind { isIronicDeployment = false } - removeIronic(ctx, func() RemoveIronicInput { - return RemoveIronicInput{ + removeIronicBMO(ctx, func() RemoveIronicBMOInput { + return RemoveIronicBMOInput{ ManagementCluster: input.BootstrapClusterProxy, IsDeployment: isIronicDeployment, Namespace: input.E2EConfig.GetVariable(ironicNamespace), NamePrefix: input.E2EConfig.GetVariable(NamePrefix), + NameSuffix: "-ironic", } }) @@ -178,6 +180,18 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) { }) LogFromFile(filepath.Join(input.ArtifactFolder, "clusters", input.ClusterName+"-bootstrap", "logs", input.Namespace, "clusterctl-move.log")) + By("Remove BMO deployment from the source cluster") + isDeployment := true + removeIronicBMO(ctx, func() RemoveIronicBMOInput { + return RemoveIronicBMOInput{ + ManagementCluster: input.BootstrapClusterProxy, + IsDeployment: isDeployment, + Namespace: input.E2EConfig.GetVariable(ironicNamespace), + NamePrefix: input.E2EConfig.GetVariable(NamePrefix), + NameSuffix: "-controller-manager", + } + }) + pivotingCluster := framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: input.TargetCluster.GetClient(), Namespace: input.Namespace, @@ -273,20 +287,21 @@ func installIronicBMO(inputGetter func() installIronicBMOInput) { Expect(err).To(BeNil(), "Failed to deploy Ironic") } -type RemoveIronicInput struct { +type RemoveIronicBMOInput struct { ManagementCluster framework.ClusterProxy IsDeployment bool Namespace string NamePrefix string + NameSuffix string } -func removeIronic(ctx context.Context, inputGetter func() RemoveIronicInput) { +func removeIronicBMO(ctx context.Context, inputGetter func() RemoveIronicBMOInput) { input := inputGetter() if input.IsDeployment { - deploymentName := input.NamePrefix + "-ironic" + deploymentName := input.NamePrefix + input.NameSuffix ironicNamespace := input.Namespace err := input.ManagementCluster.GetClientSet().AppsV1().Deployments(ironicNamespace).Delete(ctx, deploymentName, metav1.DeleteOptions{}) - Expect(err).To(BeNil(), "Failed to delete Ironic from the source cluster") + Expect(err).To(BeNil(), "Failed to delete Ironic") } else { ironicContainerList := []string{ "ironic", @@ -372,14 +387,30 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) { numberOfControlplane := int(*input.E2EConfig.GetInt32PtrVariable("CONTROL_PLANE_MACHINE_COUNT")) numberOfAllBmh := numberOfWorkers + numberOfControlplane By("Remove Ironic deployment from target cluster") - removeIronic(ctx, func() RemoveIronicInput { - return RemoveIronicInput{ + removeIronicBMO(ctx, func() RemoveIronicBMOInput { + return RemoveIronicBMOInput{ ManagementCluster: input.TargetCluster, IsDeployment: true, Namespace: input.E2EConfig.GetVariable(ironicNamespace), NamePrefix: input.E2EConfig.GetVariable(NamePrefix), + NameSuffix: "-ironic", + } + }) + + By("Reinstate BMO in Source cluster") + installIronicBMO(func() installIronicBMOInput { + return installIronicBMOInput{ + ManagementCluster: input.BootstrapClusterProxy, + BMOPath: input.E2EConfig.GetVariable(bmoPath), + deployIronic: false, + deployBMO: true, + deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)), + DeployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)), + NamePrefix: input.E2EConfig.GetVariable(NamePrefix), + RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)), } }) + By("Reinstate Ironic containers and BMH") // TODO(mboukhalfa): add this local ironic deployment case to installIronicBMO function ephemeralCluster := os.Getenv("EPHEMERAL_CLUSTER") @@ -426,6 +457,17 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) { LogFromFile(filepath.Join(input.ArtifactFolder, "clusters", input.ClusterName+"-pivot", "logs", input.Namespace, "clusterctl-move.log")) + By("Remove BMO deployment from target cluster") + removeIronicBMO(ctx, func() RemoveIronicBMOInput { + return RemoveIronicBMOInput{ + ManagementCluster: input.TargetCluster, + IsDeployment: true, + Namespace: input.E2EConfig.GetVariable(ironicNamespace), + NamePrefix: input.E2EConfig.GetVariable(NamePrefix), + NameSuffix: "-controller-manager", + } + }) + By("Check that the re-pivoted cluster is up and running") pivotingCluster := framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: input.BootstrapClusterProxy.GetClient(), diff --git a/test/e2e/upgrade_management_cluster_test.go b/test/e2e/upgrade_management_cluster_test.go index 8a8287be9b..6b291d3b51 100644 --- a/test/e2e/upgrade_management_cluster_test.go +++ b/test/e2e/upgrade_management_cluster_test.go @@ -152,12 +152,13 @@ func preInitFunc(clusterProxy framework.ClusterProxy) { if ephemeralCluster == Kind { isIronicDeployment = false } - removeIronic(ctx, func() RemoveIronicInput { - return RemoveIronicInput{ + removeIronicBMO(ctx, func() RemoveIronicBMOInput { + return RemoveIronicBMOInput{ ManagementCluster: bootstrapClusterProxy, IsDeployment: isIronicDeployment, Namespace: e2eConfig.GetVariable(ironicNamespace), NamePrefix: e2eConfig.GetVariable(NamePrefix), + NameSuffix: "-ironic", } }) // install bmo @@ -257,12 +258,13 @@ func preCleanupManagementCluster(clusterProxy framework.ClusterProxy) { }) } } - removeIronic(ctx, func() RemoveIronicInput { - return RemoveIronicInput{ + removeIronicBMO(ctx, func() RemoveIronicBMOInput { + return RemoveIronicBMOInput{ ManagementCluster: clusterProxy, IsDeployment: true, Namespace: e2eConfig.GetVariable(ironicNamespace), NamePrefix: e2eConfig.GetVariable(NamePrefix), + NameSuffix: "-ironic", } }) reInstallIronic()