From 640fe229d02e0523cd805e1f8ce9f77538aab5a0 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Thu, 9 May 2024 17:52:30 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=B1=20test:=20use=20a=20daemonset=20an?= =?UTF-8?q?d=20kubectl=20exec=20to=20provide=20images=20to=20remote=20clus?= =?UTF-8?q?ters=20(#2955)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: use a daemonset and kubectl exec to provide images to remote clusters * review fixes * review fixes * log io.Copy error * fix typo --- Makefile | 1 - hack/e2e.sh | 20 +- test/e2e/clusterctl_upgrade_test.go | 8 +- test/e2e/config/vsphere.yaml | 1 - .../remote-management/image-injection.yaml | 8 - .../main/remote-management/kustomization.yaml | 8 - test/framework/daemonset_helpers.go | 61 +++++ test/framework/image_preloading.go | 229 ++++++++++++++++++ test/go.mod | 1 + 9 files changed, 308 insertions(+), 29 deletions(-) delete mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml delete mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml create mode 100644 test/framework/daemonset_helpers.go create mode 100644 test/framework/image_preloading.go diff --git a/Makefile b/Makefile index 23ea52d7a3..98e29ab84f 100644 --- a/Makefile +++ b/Makefile @@ -367,7 +367,6 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/remote-management" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml" # for PCI passthrough template "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml" diff --git a/hack/e2e.sh b/hack/e2e.sh index a9c18bbf24..7199d084a5 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -74,7 +74,6 @@ function login() { # NOTE: when running on CI without presets, value for variables are missing: GOVC_URL, GOVC_USERNAME, GOVC_PASSWORD, VM_SSH_PUB_KEY), # but this is not an issue when we are targeting vcsim (corresponding VSPHERE_ variables will be injected during test setup). AUTH= -E2E_IMAGE_SHA= GCR_KEY_FILE="${GCR_KEY_FILE:-}" export VSPHERE_SERVER="${GOVC_URL:-}" export VSPHERE_USERNAME="${GOVC_USERNAME:-}" @@ -144,16 +143,19 @@ ARCH="$(go env GOARCH)" # Only build and upload the image if we run tests which require it to save some $. # NOTE: the image is required for clusterctl upgrade tests, and those test are run only as part of the main e2e test job (without any focus) if [[ -z "${GINKGO_FOCUS+x}" ]]; then - # Save the docker image locally + # Save the docker images locally make e2e-images mkdir -p /tmp/images - docker save "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" -o "$DOCKER_IMAGE_TAR" - - # Store the image on gcs - login - E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev") - export E2E_IMAGE_SHA - gsutil cp ${DOCKER_IMAGE_TAR} gs://capv-ci/"$E2E_IMAGE_SHA" + if [[ ${GINKGO_FOCUS:-} =~ \\\[supervisor\\\] ]]; then + docker save \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-${ARCH}:dev" \ + > ${DOCKER_IMAGE_TAR} + else + docker save \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \ + > ${DOCKER_IMAGE_TAR} + fi fi # Run e2e tests diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index c78913bdb4..56daa0caf1 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -24,6 +24,8 @@ import ( . "github.com/onsi/gomega" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework/clusterctl" + + vsphereframework "sigs.k8s.io/cluster-api-provider-vsphere/test/framework" ) var ( @@ -51,8 +53,9 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10 BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("remote-management"), + MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("topology"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)}, @@ -87,8 +90,9 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9= BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("remote-management"), + MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("topology"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)}, diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index 1e7dd2618b..fda35a13db 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -169,7 +169,6 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-node-drain.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ownerrefs-finalizers.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-pci.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-remote-management.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-storage-policy.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template.yaml" diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml deleted file mode 100644 index 0cbda95f31..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- op: add - path: /spec/topology/variables/- - value: - name: preKubeadmScript - value: | - mkdir -p /opt/cluster-api - curl "https://storage.googleapis.com/capv-ci/${E2E_IMAGE_SHA}" -o /opt/cluster-api/image.tar - ctr -n k8s.io images import /opt/cluster-api/image.tar diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml deleted file mode 100644 index a3096d07f7..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../topology -patches: - - target: - kind: Cluster - path: ./image-injection.yaml diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go new file mode 100644 index 0000000000..c4f88cdc9d --- /dev/null +++ b/test/framework/daemonset_helpers.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// waitForDaemonSetAvailableInput is the input for waitForDaemonSetAvailable. +type waitForDaemonSetAvailableInput struct { + Getter framework.Getter + Daemonset *appsv1.DaemonSet +} + +// waitForDaemonSetAvailable waits until the DaemonSet is rolled out: +// * status.updatedNumberScheduled < status.DesiredNumberScheduled. +// * status.NumberAvailable < status.DesiredNumberScheduled. +func waitForDaemonSetAvailable(ctx context.Context, input waitForDaemonSetAvailableInput, intervals ...interface{}) { + Byf("Waiting for daemonset %s to be available", klog.KObj(input.Daemonset)) + daemonSet := &appsv1.DaemonSet{} + Eventually(func() bool { + key := client.ObjectKey{ + Namespace: input.Daemonset.GetNamespace(), + Name: input.Daemonset.GetName(), + } + if err := input.Getter.Get(ctx, key, daemonSet); err != nil { + return false + } + if daemonSet.Generation <= daemonSet.Status.ObservedGeneration { + if daemonSet.Status.UpdatedNumberScheduled < daemonSet.Status.DesiredNumberScheduled { + return false + } + if daemonSet.Status.NumberAvailable < daemonSet.Status.DesiredNumberScheduled { + return false + } + return true + } + return false + }, intervals...).Should(BeTrue()) +} diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go new file mode 100644 index 0000000000..16ea35884a --- /dev/null +++ b/test/framework/image_preloading.go @@ -0,0 +1,229 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func LoadImagesFunc(ctx context.Context) func(clusterProxy framework.ClusterProxy) { + sourceFile := os.Getenv("DOCKER_IMAGE_TAR") + Expect(sourceFile).ToNot(BeEmpty(), "DOCKER_IMAGE_TAR must be set") + + return func(clusterProxy framework.ClusterProxy) { + loadImagesToCluster(ctx, sourceFile, clusterProxy) + } +} + +// loadImagesToCluster deploys a privileged daemonset and uses it to stream-load container images. +func loadImagesToCluster(ctx context.Context, sourceFile string, clusterProxy framework.ClusterProxy) { + daemonSet, daemonSetMutateFn, daemonSetLabels := getPreloadDaemonset() + ctrlClient := clusterProxy.GetClient() + + // Create the DaemonSet. + _, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemonSet, daemonSetMutateFn) + Expect(err).ToNot(HaveOccurred()) + + // Wait for DaemonSet to be available. + waitForDaemonSetAvailable(ctx, waitForDaemonSetAvailableInput{Getter: ctrlClient, Daemonset: daemonSet}, time.Minute*3, time.Second*10) + + // List all pods and load images via each found pod. + pods := &corev1.PodList{} + Expect(ctrlClient.List( + ctx, + pods, + client.InNamespace(daemonSet.Namespace), + client.MatchingLabels(daemonSetLabels), + )).To(Succeed()) + + errs := []error{} + for j := range pods.Items { + pod := pods.Items[j] + Byf("Loading images to node %s via pod %s", pod.Spec.NodeName, klog.KObj(&pod)) + if err := loadImagesViaPod(ctx, clusterProxy, sourceFile, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name); err != nil { + errs = append(errs, err) + } + } + Expect(kerrors.NewAggregate(errs)).ToNot(HaveOccurred()) + + // Delete the DaemonSet. + Expect(ctrlClient.Delete(ctx, daemonSet)).To(Succeed()) +} + +func loadImagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, sourceFile, namespace, podName, containerName string) error { + // Open source tar file. + reader, writer := io.Pipe() + file, err := os.Open(filepath.Clean(sourceFile)) + if err != nil { + return err + } + + // Use go routine to pipe source file content into then stdin. + go func(file *os.File, writer io.WriteCloser) { + defer writer.Close() + defer file.Close() + // Ignoring the error here because the execPod command should fail in case of + // failure copying over the data. + _, err := io.Copy(writer, file) + if err != nil { + fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to copy file data to io.Pipe: %v\n", err) + } + }(file, writer) + + // Load the container images using ctr and delete the file. + loadCommand := "ctr -n k8s.io images import -" + return execPod(ctx, clusterProxy, namespace, podName, containerName, loadCommand, reader) +} + +// execPod executes a command at a pod. +// xref: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubectl/pkg/cmd/exec/exec.go#L123 +func execPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName, cmd string, stdin io.Reader) error { + var hasStdin bool + if stdin != nil { + hasStdin = true + } + + req := clusterProxy.GetClientSet().CoreV1().RESTClient().Post(). + Namespace(namespace). + Resource("pods"). + Name(podName). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: []string{"/bin/sh", "-c", cmd}, + Stdin: hasStdin, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(clusterProxy.GetRESTConfig(), "POST", req.URL()) + if err != nil { + return err + } + // WebSocketExecutor must be "GET" method as described in RFC 6455 Sec. 4.1 (page 17). + websocketExec, err := remotecommand.NewWebSocketExecutor(clusterProxy.GetRESTConfig(), "GET", req.URL().String()) + if err != nil { + return err + } + exec, err = remotecommand.NewFallbackExecutor(websocketExec, exec, httpstream.IsUpgradeFailure) + if err != nil { + return err + } + + var stdout, stderr bytes.Buffer + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: &stdout, + Stderr: &stderr, + Tty: false, + }) + if err != nil { + return errors.Wrapf(err, "running command %q stdout=%q, stderr=%q", cmd, stdout.String(), stderr.String()) + } + + return nil +} + +func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[string]string) { + labels := map[string]string{ + "app": "image-preloader", + } + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: "image-preloader", + Labels: labels, + }, + } + mutateFunc := func() error { + ds.Labels = labels + ds.Spec = appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "pause", + Image: "registry.k8s.io/pause:3.9", + Command: []string{"/usr/bin/tail", "-f", "/dev/null"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host", + MountPath: "/", + }, + }, + }, + }, + HostPID: true, + HostIPC: true, + Volumes: []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + Type: ptr.To(corev1.HostPathDirectory), + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + // Tolerate any taint. + { + Operator: corev1.TolerationOpExists, + }, + }, + }, + }, + } + return nil + } + return ds, mutateFunc, labels +} diff --git a/test/go.mod b/test/go.mod index 2714ffbdba..8941b3bf87 100644 --- a/test/go.mod +++ b/test/go.mod @@ -87,6 +87,7 @@ require ( github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect