From 06106790c7b618f84b6765073ce3b90fe4a6a1af Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Tue, 30 Apr 2024 19:26:50 +0200 Subject: [PATCH 1/5] test: use a daemonset and kubectl exec to provide images to remote clusters --- Makefile | 1 - hack/e2e.sh | 20 +- test/e2e/clusterctl_upgrade_test.go | 10 +- test/e2e/config/vsphere.yaml | 1 - .../remote-management/image-injection.yaml | 8 - .../main/remote-management/kustomization.yaml | 8 - test/framework/daemonset_helpers.go | 62 +++++ test/framework/image_preloading.go | 220 ++++++++++++++++++ test/go.mod | 1 + 9 files changed, 302 insertions(+), 29 deletions(-) delete mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml delete mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml create mode 100644 test/framework/daemonset_helpers.go create mode 100644 test/framework/image_preloading.go diff --git a/Makefile b/Makefile index 23ea52d7a3..98e29ab84f 100644 --- a/Makefile +++ b/Makefile @@ -367,7 +367,6 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/remote-management" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml" # for PCI passthrough template "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml" diff --git a/hack/e2e.sh b/hack/e2e.sh index a9c18bbf24..7199d084a5 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -74,7 +74,6 @@ function login() { # NOTE: when running on CI without presets, value for variables are missing: GOVC_URL, GOVC_USERNAME, GOVC_PASSWORD, VM_SSH_PUB_KEY), # but this is not an issue when we are targeting vcsim (corresponding VSPHERE_ variables will be injected during test setup). AUTH= -E2E_IMAGE_SHA= GCR_KEY_FILE="${GCR_KEY_FILE:-}" export VSPHERE_SERVER="${GOVC_URL:-}" export VSPHERE_USERNAME="${GOVC_USERNAME:-}" @@ -144,16 +143,19 @@ ARCH="$(go env GOARCH)" # Only build and upload the image if we run tests which require it to save some $. # NOTE: the image is required for clusterctl upgrade tests, and those test are run only as part of the main e2e test job (without any focus) if [[ -z "${GINKGO_FOCUS+x}" ]]; then - # Save the docker image locally + # Save the docker images locally make e2e-images mkdir -p /tmp/images - docker save "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" -o "$DOCKER_IMAGE_TAR" - - # Store the image on gcs - login - E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev") - export E2E_IMAGE_SHA - gsutil cp ${DOCKER_IMAGE_TAR} gs://capv-ci/"$E2E_IMAGE_SHA" + if [[ ${GINKGO_FOCUS:-} =~ \\\[supervisor\\\] ]]; then + docker save \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-${ARCH}:dev" \ + > ${DOCKER_IMAGE_TAR} + else + docker save \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \ + > ${DOCKER_IMAGE_TAR} + fi fi # Run e2e tests diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index c78913bdb4..627e0d1df8 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -24,6 +24,8 @@ import ( . "github.com/onsi/gomega" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework/clusterctl" + + vsphereframework "sigs.k8s.io/cluster-api-provider-vsphere/test/framework" ) var ( @@ -51,8 +53,10 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10 BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("remote-management"), + MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("topology"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + PreInit: vsphereframework.LoadImagesFunc(ctx), + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)}, @@ -87,8 +91,10 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9= BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("remote-management"), + MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("topology"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + PreInit: vsphereframework.LoadImagesFunc(ctx), + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)}, diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index 1e7dd2618b..fda35a13db 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -169,7 +169,6 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-node-drain.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ownerrefs-finalizers.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-pci.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-remote-management.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-storage-policy.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template.yaml" diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml deleted file mode 100644 index 0cbda95f31..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- op: add - path: /spec/topology/variables/- - value: - name: preKubeadmScript - value: | - mkdir -p /opt/cluster-api - curl "https://storage.googleapis.com/capv-ci/${E2E_IMAGE_SHA}" -o /opt/cluster-api/image.tar - ctr -n k8s.io images import /opt/cluster-api/image.tar diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml deleted file mode 100644 index a3096d07f7..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../topology -patches: - - target: - kind: Cluster - path: ./image-injection.yaml diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go new file mode 100644 index 0000000000..2b11c08e25 --- /dev/null +++ b/test/framework/daemonset_helpers.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// waitForDaemonSetAvailableInput is the input for WaitForDeploymentsAvailable. +type waitForDaemonSetAvailableInput struct { + Getter framework.Getter + Daemonset *appsv1.DaemonSet +} + +// waitForDaemonSetAvailable waits until the Deployment has status.Available = True, that signals that +// all the desired replicas are in place. +// This can be used to check if Cluster API controllers installed in the management cluster are working. +// xref: https://github.com/kubernetes/kubernetes/blob/bfa4188/staging/src/k8s.io/kubectl/pkg/polymorphichelpers/rollout_status.go#L95 +func waitForDaemonSetAvailable(ctx context.Context, input waitForDaemonSetAvailableInput, intervals ...interface{}) { + Byf("Waiting for daemonset %s to be available", klog.KObj(input.Daemonset)) + daemon := &appsv1.DaemonSet{} + Eventually(func() bool { + key := client.ObjectKey{ + Namespace: input.Daemonset.GetNamespace(), + Name: input.Daemonset.GetName(), + } + if err := input.Getter.Get(ctx, key, daemon); err != nil { + return false + } + if daemon.Generation <= daemon.Status.ObservedGeneration { + if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled { + return false + } + if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled { + return false + } + return true + } + return false + }, intervals...).Should(BeTrue()) +} diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go new file mode 100644 index 0000000000..85da3855bb --- /dev/null +++ b/test/framework/image_preloading.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "context" + "io" + "os" + "time" + + . "github.com/onsi/gomega" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func LoadImagesFunc(ctx context.Context) func(clusterProxy framework.ClusterProxy) { + sourceFile := os.Getenv("DOCKER_IMAGE_TAR") + Expect(sourceFile).ToNot(BeEmpty(), "DOCKER_IMAGE_TAR must be set") + + loader := imagePreloader{ + sourceFile: sourceFile, + } + + return func(clusterProxy framework.ClusterProxy) { + loader.ImagesToCluster(ctx, clusterProxy) + } +} + +type imagePreloader struct { + sourceFile string +} + +// ImagesToCluster deploys a privileged daemonset and uses it to stream-load container images. +func (loader *imagePreloader) ImagesToCluster(ctx context.Context, clusterProxy framework.ClusterProxy) { + daemon, daemonMutateFn, daemonLabels := getPreloadDaemonset() + ctrlClient := clusterProxy.GetClient() + + // Create Daemonset + _, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemon, daemonMutateFn) + Expect(err).ToNot(HaveOccurred()) + + // Wait for DaemonSet to be available. + waitForDaemonSetAvailable(ctx, waitForDaemonSetAvailableInput{Getter: ctrlClient, Daemonset: daemon}, time.Minute*3, time.Second*10) + + // List all pods and load images via each found pod. + pods := &corev1.PodList{} + Expect(ctrlClient.List( + ctx, + pods, + client.InNamespace(daemon.Namespace), + client.MatchingLabels(daemonLabels), + )).To(Succeed()) + + errs := []error{} + for j := range pods.Items { + pod := pods.Items[j] + Byf("Loading images to node %s via pod %s", pod.Spec.NodeName, klog.KObj(&pod)) + if err := loader.imagesViaPod(ctx, clusterProxy, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name); err != nil { + errs = append(errs, err) + } + } + + Expect(kerrors.NewAggregate(errs)).ToNot(HaveOccurred()) +} + +func (loader *imagePreloader) imagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName string) error { + // Open source tar file. + reader, writer := io.Pipe() + file, err := os.Open(loader.sourceFile) + if err != nil { + return err + } + + // Use go routine to pipe source file content into then stdin. + go func(file *os.File, writer io.WriteCloser) { + defer writer.Close() + defer file.Close() + _, _ = io.Copy(writer, file) + }(file, writer) + + // Load the container images using ctr and delete the file. + loadCommand := "ctr -n k8s.io images import -" + return loader.execPod(ctx, clusterProxy, namespace, podName, containerName, loadCommand, reader) +} + +func (loader *imagePreloader) execPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName, cmd string, stdin io.Reader) error { + var hasStdin bool + if stdin != nil { + hasStdin = true + } + + req := clusterProxy.GetClientSet().CoreV1().RESTClient().Post(). + Namespace(namespace). + Resource("pods"). + Name(podName). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: []string{"/bin/sh", "-c", cmd}, + Stdin: hasStdin, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(clusterProxy.GetRESTConfig(), "POST", req.URL()) + if err != nil { + return err + } + // WebSocketExecutor must be "GET" method as described in RFC 6455 Sec. 4.1 (page 17). + websocketExec, err := remotecommand.NewWebSocketExecutor(clusterProxy.GetRESTConfig(), "GET", req.URL().String()) + if err != nil { + return err + } + exec, err = remotecommand.NewFallbackExecutor(websocketExec, exec, httpstream.IsUpgradeFailure) + if err != nil { + return err + } + + var stdout, stderr bytes.Buffer + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: &stdout, + Stderr: &stderr, + Tty: false, + }) + + if err != nil { + return errors.Wrapf(err, "running command %q stdout=%q, stderr=%q", cmd, stdout.String(), stderr.String()) + } + + return nil +} + +func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[string]string) { + labels := map[string]string{ + "app": "image-preloader", + } + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: "image-preloader", + Labels: labels, + }, + } + muatetFn := func() error { + ds.Labels = labels + ds.Spec = appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "pause", + Image: "registry.k8s.io/pause:3.9", + Command: []string{"/usr/bin/tail", "-f", "/dev/null"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host", + MountPath: "/", + }, + }, + }, + }, + HostPID: true, + HostIPC: true, + Volumes: []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + Type: ptr.To(corev1.HostPathDirectory), + }, + }, + }, + }, + }, + }, + } + return nil + } + return ds, muatetFn, labels +} diff --git a/test/go.mod b/test/go.mod index a05e034abb..a8c46efd92 100644 --- a/test/go.mod +++ b/test/go.mod @@ -87,6 +87,7 @@ require ( github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect From 02f1b6f1940122e2ebbb59a2e43818eb87d53bf7 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Mon, 6 May 2024 18:57:46 +0200 Subject: [PATCH 2/5] review fixes --- test/e2e/clusterctl_upgrade_test.go | 1 - test/framework/daemonset_helpers.go | 19 ++++++------- test/framework/image_preloading.go | 44 +++++++++++++---------------- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 627e0d1df8..57ef715729 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -93,7 +93,6 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9= SkipCleanup: skipCleanup, MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("topology"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, - PreInit: vsphereframework.LoadImagesFunc(ctx), PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go index 2b11c08e25..c4f88cdc9d 100644 --- a/test/framework/daemonset_helpers.go +++ b/test/framework/daemonset_helpers.go @@ -27,32 +27,31 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// waitForDaemonSetAvailableInput is the input for WaitForDeploymentsAvailable. +// waitForDaemonSetAvailableInput is the input for waitForDaemonSetAvailable. type waitForDaemonSetAvailableInput struct { Getter framework.Getter Daemonset *appsv1.DaemonSet } -// waitForDaemonSetAvailable waits until the Deployment has status.Available = True, that signals that -// all the desired replicas are in place. -// This can be used to check if Cluster API controllers installed in the management cluster are working. -// xref: https://github.com/kubernetes/kubernetes/blob/bfa4188/staging/src/k8s.io/kubectl/pkg/polymorphichelpers/rollout_status.go#L95 +// waitForDaemonSetAvailable waits until the DaemonSet is rolled out: +// * status.updatedNumberScheduled < status.DesiredNumberScheduled. +// * status.NumberAvailable < status.DesiredNumberScheduled. func waitForDaemonSetAvailable(ctx context.Context, input waitForDaemonSetAvailableInput, intervals ...interface{}) { Byf("Waiting for daemonset %s to be available", klog.KObj(input.Daemonset)) - daemon := &appsv1.DaemonSet{} + daemonSet := &appsv1.DaemonSet{} Eventually(func() bool { key := client.ObjectKey{ Namespace: input.Daemonset.GetNamespace(), Name: input.Daemonset.GetName(), } - if err := input.Getter.Get(ctx, key, daemon); err != nil { + if err := input.Getter.Get(ctx, key, daemonSet); err != nil { return false } - if daemon.Generation <= daemon.Status.ObservedGeneration { - if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled { + if daemonSet.Generation <= daemonSet.Status.ObservedGeneration { + if daemonSet.Status.UpdatedNumberScheduled < daemonSet.Status.DesiredNumberScheduled { return false } - if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled { + if daemonSet.Status.NumberAvailable < daemonSet.Status.DesiredNumberScheduled { return false } return true diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go index 85da3855bb..f0c7e98d0d 100644 --- a/test/framework/image_preloading.go +++ b/test/framework/image_preloading.go @@ -21,6 +21,7 @@ import ( "context" "io" "os" + "path/filepath" "time" . "github.com/onsi/gomega" @@ -44,45 +45,37 @@ func LoadImagesFunc(ctx context.Context) func(clusterProxy framework.ClusterProx sourceFile := os.Getenv("DOCKER_IMAGE_TAR") Expect(sourceFile).ToNot(BeEmpty(), "DOCKER_IMAGE_TAR must be set") - loader := imagePreloader{ - sourceFile: sourceFile, - } - return func(clusterProxy framework.ClusterProxy) { - loader.ImagesToCluster(ctx, clusterProxy) + loadImagesToCluster(ctx, sourceFile, clusterProxy) } } -type imagePreloader struct { - sourceFile string -} - -// ImagesToCluster deploys a privileged daemonset and uses it to stream-load container images. -func (loader *imagePreloader) ImagesToCluster(ctx context.Context, clusterProxy framework.ClusterProxy) { - daemon, daemonMutateFn, daemonLabels := getPreloadDaemonset() +// loadImagesToCluster deploys a privileged daemonset and uses it to stream-load container images. +func loadImagesToCluster(ctx context.Context, sourceFile string, clusterProxy framework.ClusterProxy) { + daemonSet, daemonSetMutateFn, daemonSetLabels := getPreloadDaemonset() ctrlClient := clusterProxy.GetClient() // Create Daemonset - _, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemon, daemonMutateFn) + _, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemonSet, daemonSetMutateFn) Expect(err).ToNot(HaveOccurred()) // Wait for DaemonSet to be available. - waitForDaemonSetAvailable(ctx, waitForDaemonSetAvailableInput{Getter: ctrlClient, Daemonset: daemon}, time.Minute*3, time.Second*10) + waitForDaemonSetAvailable(ctx, waitForDaemonSetAvailableInput{Getter: ctrlClient, Daemonset: daemonSet}, time.Minute*3, time.Second*10) // List all pods and load images via each found pod. pods := &corev1.PodList{} Expect(ctrlClient.List( ctx, pods, - client.InNamespace(daemon.Namespace), - client.MatchingLabels(daemonLabels), + client.InNamespace(daemonSet.Namespace), + client.MatchingLabels(daemonSetLabels), )).To(Succeed()) errs := []error{} for j := range pods.Items { pod := pods.Items[j] Byf("Loading images to node %s via pod %s", pod.Spec.NodeName, klog.KObj(&pod)) - if err := loader.imagesViaPod(ctx, clusterProxy, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name); err != nil { + if err := loadImagesViaPod(ctx, clusterProxy, sourceFile, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name); err != nil { errs = append(errs, err) } } @@ -90,10 +83,10 @@ func (loader *imagePreloader) ImagesToCluster(ctx context.Context, clusterProxy Expect(kerrors.NewAggregate(errs)).ToNot(HaveOccurred()) } -func (loader *imagePreloader) imagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName string) error { +func loadImagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, sourceFile, namespace, podName, containerName string) error { // Open source tar file. reader, writer := io.Pipe() - file, err := os.Open(loader.sourceFile) + file, err := os.Open(filepath.Clean(sourceFile)) if err != nil { return err } @@ -102,15 +95,19 @@ func (loader *imagePreloader) imagesViaPod(ctx context.Context, clusterProxy fra go func(file *os.File, writer io.WriteCloser) { defer writer.Close() defer file.Close() + // Ignoring the error here because the execPod command should fail in case of + // failure copying over the data. _, _ = io.Copy(writer, file) }(file, writer) // Load the container images using ctr and delete the file. loadCommand := "ctr -n k8s.io images import -" - return loader.execPod(ctx, clusterProxy, namespace, podName, containerName, loadCommand, reader) + return execPod(ctx, clusterProxy, namespace, podName, containerName, loadCommand, reader) } -func (loader *imagePreloader) execPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName, cmd string, stdin io.Reader) error { +// execPod executes a command at a pod. +// xref: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubectl/pkg/cmd/exec/exec.go#L123 +func execPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName, cmd string, stdin io.Reader) error { var hasStdin bool if stdin != nil { hasStdin = true @@ -152,7 +149,6 @@ func (loader *imagePreloader) execPod(ctx context.Context, clusterProxy framewor Stderr: &stderr, Tty: false, }) - if err != nil { return errors.Wrapf(err, "running command %q stdout=%q, stderr=%q", cmd, stdout.String(), stderr.String()) } @@ -171,7 +167,7 @@ func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[stri Labels: labels, }, } - muatetFn := func() error { + muatetFunc := func() error { ds.Labels = labels ds.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -216,5 +212,5 @@ func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[stri } return nil } - return ds, muatetFn, labels + return ds, muatetFunc, labels } From b171a08ef7b843b75fd9c2195d60a20b84ecc1c5 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Tue, 7 May 2024 10:29:20 +0200 Subject: [PATCH 3/5] review fixes --- test/e2e/clusterctl_upgrade_test.go | 1 - test/framework/image_preloading.go | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 57ef715729..56daa0caf1 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -55,7 +55,6 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10 SkipCleanup: skipCleanup, MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("topology"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, - PreInit: vsphereframework.LoadImagesFunc(ctx), PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go index f0c7e98d0d..5d12c86bb5 100644 --- a/test/framework/image_preloading.go +++ b/test/framework/image_preloading.go @@ -55,7 +55,7 @@ func loadImagesToCluster(ctx context.Context, sourceFile string, clusterProxy fr daemonSet, daemonSetMutateFn, daemonSetLabels := getPreloadDaemonset() ctrlClient := clusterProxy.GetClient() - // Create Daemonset + // Create the DaemonSet. _, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemonSet, daemonSetMutateFn) Expect(err).ToNot(HaveOccurred()) @@ -79,8 +79,10 @@ func loadImagesToCluster(ctx context.Context, sourceFile string, clusterProxy fr errs = append(errs, err) } } - Expect(kerrors.NewAggregate(errs)).ToNot(HaveOccurred()) + + // Delete the DaemonSet. + Expect(ctrlClient.Delete(ctx, daemonSet)).To(Succeed()) } func loadImagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, sourceFile, namespace, podName, containerName string) error { @@ -207,6 +209,12 @@ func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[stri }, }, }, + Tolerations: []corev1.Toleration{ + // Tolerate any taint. + { + Operator: corev1.TolerationOpExists, + }, + }, }, }, } From b2a8646072beb3a3c19bad681719ccf3f1b158a0 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Tue, 7 May 2024 10:33:44 +0200 Subject: [PATCH 4/5] log io.Copy error --- test/framework/image_preloading.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go index 5d12c86bb5..87b43054ac 100644 --- a/test/framework/image_preloading.go +++ b/test/framework/image_preloading.go @@ -19,11 +19,13 @@ package framework import ( "bytes" "context" + "fmt" "io" "os" "path/filepath" "time" + "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" @@ -99,7 +101,10 @@ func loadImagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, defer file.Close() // Ignoring the error here because the execPod command should fail in case of // failure copying over the data. - _, _ = io.Copy(writer, file) + _, err := io.Copy(writer, file) + if err != nil { + fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to copy file data to io.Pipe: %v\n", err) + } }(file, writer) // Load the container images using ctr and delete the file. From a8c11c150f38eb96d75410e1e1b4f7a6c26ce92e Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Tue, 7 May 2024 10:42:11 +0200 Subject: [PATCH 5/5] fix typo --- test/framework/image_preloading.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go index 87b43054ac..16ea35884a 100644 --- a/test/framework/image_preloading.go +++ b/test/framework/image_preloading.go @@ -174,7 +174,7 @@ func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[stri Labels: labels, }, } - muatetFunc := func() error { + mutateFunc := func() error { ds.Labels = labels ds.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -225,5 +225,5 @@ func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[stri } return nil } - return ds, muatetFunc, labels + return ds, mutateFunc, labels }