🌱 test: use a daemonset and kubectl exec to provide images to remote clusters

Makefile

@@ -367,7 +367,6 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml"
 	cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml"
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/remote-management" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml"
 	# for PCI passthrough template
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml"

hack/e2e.sh

@@ -74,7 +74,6 @@ function login() {
 # NOTE: when running on CI without presets, value for variables are missing: GOVC_URL, GOVC_USERNAME, GOVC_PASSWORD, VM_SSH_PUB_KEY),
 #  but this is not an issue when we are targeting vcsim (corresponding VSPHERE_ variables will be injected during test setup).
 AUTH=
-E2E_IMAGE_SHA=
 GCR_KEY_FILE="${GCR_KEY_FILE:-}"
 export VSPHERE_SERVER="${GOVC_URL:-}"
 export VSPHERE_USERNAME="${GOVC_USERNAME:-}"
@@ -144,16 +143,19 @@ ARCH="$(go env GOARCH)"
 # Only build and upload the image if we run tests which require it to save some $.
 # NOTE: the image is required for clusterctl upgrade tests, and those test are run only as part of the main e2e test job (without any focus)
 if [[ -z "${GINKGO_FOCUS+x}" ]]; then
-  # Save the docker image locally
+  # Save the docker images locally
   make e2e-images
   mkdir -p /tmp/images
-  docker save "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" -o "$DOCKER_IMAGE_TAR"
-
-  # Store the image on gcs
-  login
-  E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev")
-  export E2E_IMAGE_SHA
-  gsutil cp ${DOCKER_IMAGE_TAR} gs://capv-ci/"$E2E_IMAGE_SHA"
+  if [[ ${GINKGO_FOCUS:-} =~ \\\[supervisor\\\] ]]; then
+    docker save \
+      "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \
+      "gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-${ARCH}:dev" \
+      > ${DOCKER_IMAGE_TAR}
+  else
+    docker save \
+      "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \
+      > ${DOCKER_IMAGE_TAR}
+  fi
 fi
 
 # Run e2e tests

test/e2e/clusterctl_upgrade_test.go

@@ -24,6 +24,8 @@ import (
 	. "github.com/onsi/gomega"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+
+	vsphereframework "sigs.k8s.io/cluster-api-provider-vsphere/test/framework"
 )
 
 var (
@@ -51,8 +53,9 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10
 				BootstrapClusterProxy:             bootstrapClusterProxy,
 				ArtifactFolder:                    artifactFolder,
 				SkipCleanup:                       skipCleanup,
-				MgmtFlavor:                        testSpecificSettingsGetter().FlavorForMode("remote-management"),
+				MgmtFlavor:                        testSpecificSettingsGetter().FlavorForMode("topology"),
 				PostNamespaceCreated:              testSpecificSettingsGetter().PostNamespaceCreatedFunc,
+				PreUpgrade:                        vsphereframework.LoadImagesFunc(ctx),
 				InitWithBinary:                    fmt.Sprintf(clusterctlDownloadURL, capiStableRelease),
 				InitWithCoreProvider:              fmt.Sprintf(providerCAPIPrefix, capiStableRelease),
 				InitWithBootstrapProviders:        []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)},
@@ -87,8 +90,9 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=
 				BootstrapClusterProxy:             bootstrapClusterProxy,
 				ArtifactFolder:                    artifactFolder,
 				SkipCleanup:                       skipCleanup,
-				MgmtFlavor:                        testSpecificSettingsGetter().FlavorForMode("remote-management"),
+				MgmtFlavor:                        testSpecificSettingsGetter().FlavorForMode("topology"),
 				PostNamespaceCreated:              testSpecificSettingsGetter().PostNamespaceCreatedFunc,
+				PreUpgrade:                        vsphereframework.LoadImagesFunc(ctx),
 				InitWithBinary:                    fmt.Sprintf(clusterctlDownloadURL, capiStableRelease),
 				InitWithCoreProvider:              fmt.Sprintf(providerCAPIPrefix, capiStableRelease),
 				InitWithBootstrapProviders:        []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)},

test/e2e/config/vsphere.yaml

@@ -169,7 +169,6 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-node-drain.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ownerrefs-finalizers.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-pci.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-remote-management.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-storage-policy.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template.yaml"

test/framework/daemonset_helpers.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"context"
+
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework"
+	. "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// waitForDaemonSetAvailableInput is the input for waitForDaemonSetAvailable.
+type waitForDaemonSetAvailableInput struct {
+	Getter    framework.Getter
+	Daemonset *appsv1.DaemonSet
+}
+
+// waitForDaemonSetAvailable waits until the DaemonSet is rolled out:
+// * status.updatedNumberScheduled < status.DesiredNumberScheduled.
+// * status.NumberAvailable < status.DesiredNumberScheduled.
+func waitForDaemonSetAvailable(ctx context.Context, input waitForDaemonSetAvailableInput, intervals ...interface{}) {
+	Byf("Waiting for daemonset %s to be available", klog.KObj(input.Daemonset))
+	daemonSet := &appsv1.DaemonSet{}
+	Eventually(func() bool {
+		key := client.ObjectKey{
+			Namespace: input.Daemonset.GetNamespace(),
+			Name:      input.Daemonset.GetName(),
+		}
+		if err := input.Getter.Get(ctx, key, daemonSet); err != nil {
+			return false
+		}
+		if daemonSet.Generation <= daemonSet.Status.ObservedGeneration {
+			if daemonSet.Status.UpdatedNumberScheduled < daemonSet.Status.DesiredNumberScheduled {
+				return false
+			}
+			if daemonSet.Status.NumberAvailable < daemonSet.Status.DesiredNumberScheduled {
+				return false
+			}
+			return true
+		}
+		return false
+	}, intervals...).Should(BeTrue())
+}

test/framework/image_preloading.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/cluster-api/test/framework"
+	. "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func LoadImagesFunc(ctx context.Context) func(clusterProxy framework.ClusterProxy) {
+	sourceFile := os.Getenv("DOCKER_IMAGE_TAR")
+	Expect(sourceFile).ToNot(BeEmpty(), "DOCKER_IMAGE_TAR must be set")
+
+	return func(clusterProxy framework.ClusterProxy) {
+		loadImagesToCluster(ctx, sourceFile, clusterProxy)
+	}
+}
+
+// loadImagesToCluster deploys a privileged daemonset and uses it to stream-load container images.
+func loadImagesToCluster(ctx context.Context, sourceFile string, clusterProxy framework.ClusterProxy) {
+	daemonSet, daemonSetMutateFn, daemonSetLabels := getPreloadDaemonset()
+	ctrlClient := clusterProxy.GetClient()
+
+	// Create the DaemonSet.
+	_, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemonSet, daemonSetMutateFn)
+	Expect(err).ToNot(HaveOccurred())
+
+	// Wait for DaemonSet to be available.
+	waitForDaemonSetAvailable(ctx, waitForDaemonSetAvailableInput{Getter: ctrlClient, Daemonset: daemonSet}, time.Minute*3, time.Second*10)
+
+	// List all pods and load images via each found pod.
+	pods := &corev1.PodList{}
+	Expect(ctrlClient.List(
+		ctx,
+		pods,
+		client.InNamespace(daemonSet.Namespace),
+		client.MatchingLabels(daemonSetLabels),
+	)).To(Succeed())
+
+	errs := []error{}
+	for j := range pods.Items {
+		pod := pods.Items[j]
+		Byf("Loading images to node %s via pod %s", pod.Spec.NodeName, klog.KObj(&pod))
+		if err := loadImagesViaPod(ctx, clusterProxy, sourceFile, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	Expect(kerrors.NewAggregate(errs)).ToNot(HaveOccurred())
+
+	// Delete the DaemonSet.
+	Expect(ctrlClient.Delete(ctx, daemonSet)).To(Succeed())
+}
+
+func loadImagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, sourceFile, namespace, podName, containerName string) error {
+	// Open source tar file.
+	reader, writer := io.Pipe()
+	file, err := os.Open(filepath.Clean(sourceFile))
+	if err != nil {
+		return err
+	}
+
+	// Use go routine to pipe source file content into then stdin.
+	go func(file *os.File, writer io.WriteCloser) {
+		defer writer.Close()
+		defer file.Close()
+		// Ignoring the error here because the execPod command should fail in case of
+		// failure copying over the data.
+		_, err := io.Copy(writer, file)
+		if err != nil {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to copy file data to io.Pipe: %v\n", err)
+		}
+	}(file, writer)
+
+	// Load the container images using ctr and delete the file.
+	loadCommand := "ctr -n k8s.io images import -"
+	return execPod(ctx, clusterProxy, namespace, podName, containerName, loadCommand, reader)
+}
+
+// execPod executes a command at a pod.
+// xref: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubectl/pkg/cmd/exec/exec.go#L123
+func execPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName, cmd string, stdin io.Reader) error {
+	var hasStdin bool
+	if stdin != nil {
+		hasStdin = true
+	}
+
+	req := clusterProxy.GetClientSet().CoreV1().RESTClient().Post().
+		Namespace(namespace).
+		Resource("pods").
+		Name(podName).
+		SubResource("exec").
+		VersionedParams(&corev1.PodExecOptions{
+			Container: containerName,
+			Command:   []string{"/bin/sh", "-c", cmd},
+			Stdin:     hasStdin,
+			Stdout:    true,
+			Stderr:    true,
+			TTY:       false,
+		}, scheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(clusterProxy.GetRESTConfig(), "POST", req.URL())
+	if err != nil {
+		return err
+	}
+	// WebSocketExecutor must be "GET" method as described in RFC 6455 Sec. 4.1 (page 17).
+	websocketExec, err := remotecommand.NewWebSocketExecutor(clusterProxy.GetRESTConfig(), "GET", req.URL().String())
+	if err != nil {
+		return err
+	}
+	exec, err = remotecommand.NewFallbackExecutor(websocketExec, exec, httpstream.IsUpgradeFailure)
+	if err != nil {
+		return err
+	}
+
+	var stdout, stderr bytes.Buffer
+
+	err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+		Stdin:  stdin,
+		Stdout: &stdout,
+		Stderr: &stderr,
+		Tty:    false,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "running command %q stdout=%q, stderr=%q", cmd, stdout.String(), stderr.String())
+	}
+
+	return nil
+}
+
+func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[string]string) {
+	labels := map[string]string{
+		"app": "image-preloader",
+	}
+	ds := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: metav1.NamespaceSystem,
+			Name:      "image-preloader",
+			Labels:    labels,
+		},
+	}
+	mutateFunc := func() error {
+		ds.Labels = labels
+		ds.Spec = appsv1.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:    "pause",
+							Image:   "registry.k8s.io/pause:3.9",
+							Command: []string{"/usr/bin/tail", "-f", "/dev/null"},
+							SecurityContext: &corev1.SecurityContext{
+								Privileged: ptr.To(true),
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "host",
+									MountPath: "/",
+								},
+							},
+						},
+					},
+					HostPID: true,
+					HostIPC: true,
+					Volumes: []corev1.Volume{
+						{
+							Name: "host",
+							VolumeSource: corev1.VolumeSource{
+								HostPath: &corev1.HostPathVolumeSource{
+									Path: "/",
+									Type: ptr.To(corev1.HostPathDirectory),
+								},
+							},
+						},
+					},
+					Tolerations: []corev1.Toleration{
+						// Tolerate any taint.
+						{
+							Operator: corev1.TolerationOpExists,
+						},
+					},
+				},
+			},
+		}
+		return nil
+	}
+	return ds, mutateFunc, labels
+}