diff --git a/Makefile b/Makefile index 23ea52d7a3..98e29ab84f 100644 --- a/Makefile +++ b/Makefile @@ -367,7 +367,6 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/remote-management" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml" # for PCI passthrough template "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml" diff --git a/hack/e2e.sh b/hack/e2e.sh index a9c18bbf24..7199d084a5 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -74,7 +74,6 @@ function login() { # NOTE: when running on CI without presets, value for variables are missing: GOVC_URL, GOVC_USERNAME, GOVC_PASSWORD, VM_SSH_PUB_KEY), # but this is not an issue when we are targeting vcsim (corresponding VSPHERE_ variables will be injected during test setup). AUTH= -E2E_IMAGE_SHA= GCR_KEY_FILE="${GCR_KEY_FILE:-}" export VSPHERE_SERVER="${GOVC_URL:-}" export VSPHERE_USERNAME="${GOVC_USERNAME:-}" @@ -144,16 +143,19 @@ ARCH="$(go env GOARCH)" # Only build and upload the image if we run tests which require it to save some $. # NOTE: the image is required for clusterctl upgrade tests, and those test are run only as part of the main e2e test job (without any focus) if [[ -z "${GINKGO_FOCUS+x}" ]]; then - # Save the docker image locally + # Save the docker images locally make e2e-images mkdir -p /tmp/images - docker save "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" -o "$DOCKER_IMAGE_TAR" - - # Store the image on gcs - login - E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev") - export E2E_IMAGE_SHA - gsutil cp ${DOCKER_IMAGE_TAR} gs://capv-ci/"$E2E_IMAGE_SHA" + if [[ ${GINKGO_FOCUS:-} =~ \\\[supervisor\\\] ]]; then + docker save \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-${ARCH}:dev" \ + > ${DOCKER_IMAGE_TAR} + else + docker save \ + "gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-${ARCH}:dev" \ + > ${DOCKER_IMAGE_TAR} + fi fi # Run e2e tests diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index c78913bdb4..20b1ac50ba 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + vsphereframework "sigs.k8s.io/cluster-api-provider-vsphere/test/framework" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) @@ -51,8 +52,10 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10 BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("remote-management"), + MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("topology"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + PreInit: vsphereframework.LoadImagesFunc(ctx), + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)}, @@ -89,6 +92,8 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9= SkipCleanup: skipCleanup, MgmtFlavor: testSpecificSettingsGetter().FlavorForMode("remote-management"), PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + PreInit: vsphereframework.LoadImagesFunc(ctx), + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, capiStableRelease), InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, capiStableRelease), InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, capiStableRelease)}, diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index 1e7dd2618b..fda35a13db 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -169,7 +169,6 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-node-drain.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ownerrefs-finalizers.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-pci.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-remote-management.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-storage-policy.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template.yaml" diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml deleted file mode 100644 index 0cbda95f31..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/image-injection.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- op: add - path: /spec/topology/variables/- - value: - name: preKubeadmScript - value: | - mkdir -p /opt/cluster-api - curl "https://storage.googleapis.com/capv-ci/${E2E_IMAGE_SHA}" -o /opt/cluster-api/image.tar - ctr -n k8s.io images import /opt/cluster-api/image.tar diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml deleted file mode 100644 index a3096d07f7..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/remote-management/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../topology -patches: - - target: - kind: Cluster - path: ./image-injection.yaml diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go new file mode 100644 index 0000000000..2b11c08e25 --- /dev/null +++ b/test/framework/daemonset_helpers.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// waitForDaemonSetAvailableInput is the input for WaitForDeploymentsAvailable. +type waitForDaemonSetAvailableInput struct { + Getter framework.Getter + Daemonset *appsv1.DaemonSet +} + +// waitForDaemonSetAvailable waits until the Deployment has status.Available = True, that signals that +// all the desired replicas are in place. +// This can be used to check if Cluster API controllers installed in the management cluster are working. +// xref: https://github.com/kubernetes/kubernetes/blob/bfa4188/staging/src/k8s.io/kubectl/pkg/polymorphichelpers/rollout_status.go#L95 +func waitForDaemonSetAvailable(ctx context.Context, input waitForDaemonSetAvailableInput, intervals ...interface{}) { + Byf("Waiting for daemonset %s to be available", klog.KObj(input.Daemonset)) + daemon := &appsv1.DaemonSet{} + Eventually(func() bool { + key := client.ObjectKey{ + Namespace: input.Daemonset.GetNamespace(), + Name: input.Daemonset.GetName(), + } + if err := input.Getter.Get(ctx, key, daemon); err != nil { + return false + } + if daemon.Generation <= daemon.Status.ObservedGeneration { + if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled { + return false + } + if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled { + return false + } + return true + } + return false + }, intervals...).Should(BeTrue()) +} diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go new file mode 100644 index 0000000000..957e3d1d18 --- /dev/null +++ b/test/framework/image_preloading.go @@ -0,0 +1,219 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "context" + "io" + "os" + "time" + + . "github.com/onsi/gomega" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func LoadImagesFunc(ctx context.Context) func(clusterProxy framework.ClusterProxy) { + sourceFile := os.Getenv("DOCKER_IMAGE_TAR") + Expect(sourceFile).ToNot(BeEmpty(), "DOCKER_IMAGE_TAR must be set") + + loader := imagePreloader{ + sourceFile: sourceFile, + } + + return func(clusterProxy framework.ClusterProxy) { + loader.loadImagesToCluster(ctx, clusterProxy) + } +} + +type imagePreloader struct { + sourceFile string +} + +// loadImagesToCluster deploys a privileged daemonset and uses it to stream-load container images. +func (i imagePreloader) loadImagesToCluster(ctx context.Context, clusterProxy framework.ClusterProxy) { + daemon, daemonMutateFn, daemonLabels := getPreloadDaemonset() + ctrlClient := clusterProxy.GetClient() + + // Create Daemonset + _, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemon, daemonMutateFn) + Expect(err).ToNot(HaveOccurred()) + + // Wait for DaemonSet to be available. + waitForDaemonSetAvailable(ctx, waitForDaemonSetAvailableInput{Getter: ctrlClient, Daemonset: daemon}, time.Minute*3, time.Second*10) + + // List all pods and load images via each found pod. + pods := &corev1.PodList{} + Expect(ctrlClient.List( + ctx, + pods, + client.InNamespace(daemon.Namespace), + client.MatchingLabels(daemonLabels), + )).To(Succeed()) + + errs := []error{} + for _, pod := range pods.Items { + Byf("Loading images to node %s via pod %s", &pod.Spec.NodeName, klog.KObj(&pod)) + if err := i.loadImagesViaPod(ctx, clusterProxy, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name); err != nil { + errs = append(errs, err) + } + } + + Expect(kerrors.NewAggregate(errs)).ToNot(HaveOccurred()) +} + +func (i imagePreloader) loadImagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName string) error { + // Open source tar file. + reader, writer := io.Pipe() + file, err := os.Open(i.sourceFile) + if err != nil { + return err + } + + // Use go routine to pipe source file content into then stdin. + go func(file *os.File, writer io.WriteCloser) { + defer writer.Close() + defer file.Close() + _, _ = io.Copy(writer, file) + }(file, writer) + + // Load the container images using ctr and delete the file. + loadCommand := "ctr -n k8s.io images import -" + return i.execPod(ctx, clusterProxy, namespace, podName, containerName, loadCommand, reader) +} + +func (i imagePreloader) execPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName, cmd string, stdin io.Reader) error { + var hasStdin bool + if stdin != nil { + hasStdin = true + } + + req := clusterProxy.GetClientSet().CoreV1().RESTClient().Post(). + Namespace(namespace). + Resource("pods"). + Name(podName). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: []string{"/bin/sh", "-c", cmd}, + Stdin: hasStdin, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(clusterProxy.GetRESTConfig(), "POST", req.URL()) + if err != nil { + return err + } + // WebSocketExecutor must be "GET" method as described in RFC 6455 Sec. 4.1 (page 17). + websocketExec, err := remotecommand.NewWebSocketExecutor(clusterProxy.GetRESTConfig(), "GET", req.URL().String()) + if err != nil { + return err + } + exec, err = remotecommand.NewFallbackExecutor(websocketExec, exec, httpstream.IsUpgradeFailure) + if err != nil { + return err + } + + var stdout, stderr bytes.Buffer + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: &stdout, + Stderr: &stderr, + Tty: false, + }) + + if err != nil { + return errors.Wrapf(err, "running command %q stdout=%q, stderr=%q", cmd, stdout.String(), stderr.String()) + } + + return nil +} + +func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[string]string) { + labels := map[string]string{ + "app": "image-preloader", + } + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: "image-preloader", + Labels: labels, + }, + } + muatetFn := func() error { + ds.Labels = labels + ds.Spec = appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "pause", + Image: "registry.k8s.io/pause:3.9", + Command: []string{"/usr/bin/tail", "-f", "/dev/null"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host", + MountPath: "/", + }, + }, + }, + }, + HostPID: true, + HostIPC: true, + Volumes: []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + Type: ptr.To(corev1.HostPathDirectory), + }, + }, + }, + }, + }, + }, + } + return nil + } + return ds, muatetFn, labels +} diff --git a/test/go.mod b/test/go.mod index a05e034abb..a8c46efd92 100644 --- a/test/go.mod +++ b/test/go.mod @@ -87,6 +87,7 @@ require ( github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect