diff --git a/.golangci.yml b/.golangci.yml index d21a5073cf..507d8ffa6f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -230,6 +230,11 @@ issues: - stylecheck path: test/e2e/.*.go text: should not use dot imports + - linters: + - revive + - stylecheck + path: test/framework/.*.go + text: should not use dot imports - linters: - revive - stylecheck diff --git a/Makefile b/Makefile index 857f681aaf..88ba4f1893 100644 --- a/Makefile +++ b/Makefile @@ -321,7 +321,6 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/topology" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-topology.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/remote-management" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml" # for PCI passthrough template "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/pci" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-pci.yaml" diff --git a/hack/e2e.sh b/hack/e2e.sh index 57dd6038fc..bca9124694 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -71,7 +71,6 @@ function login() { } AUTH= -E2E_IMAGE_SHA= GCR_KEY_FILE="${GCR_KEY_FILE:-}" export VSPHERE_SERVER="${GOVC_URL}" export VSPHERE_USERNAME="${GOVC_USERNAME}" @@ -144,16 +143,12 @@ echo "Acquired Workload Cluster Control Plane IP: $WORKLOAD_CONTROL_PLANE_ENDPOI # Only build and upload the image if we run tests which require it to save some $. if [[ -z "${GINKGO_FOCUS+x}" ]]; then - # save the docker image locally - make e2e-image + # Save the docker images locally + make e2e-images mkdir -p /tmp/images - docker save gcr.io/k8s-staging-cluster-api/capv-manager:e2e -o "$DOCKER_IMAGE_TAR" - - # store the image on gcs - login - E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' gcr.io/k8s-staging-cluster-api/capv-manager:e2e) - export E2E_IMAGE_SHA - gsutil cp ${DOCKER_IMAGE_TAR} gs://capv-ci/"$E2E_IMAGE_SHA" + docker save \ + "gcr.io/k8s-staging-cluster-api/capv-manager:e2e" \ + > ${DOCKER_IMAGE_TAR} fi # Run e2e tests diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 063e297d5e..45ed636c32 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -19,6 +19,8 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + + vsphereframework "sigs.k8s.io/cluster-api-provider-vsphere/test/framework" ) var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.8=>current, CAPI 1.5=>1.6) [ClusterClass]", func() { @@ -29,7 +31,8 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.8= BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - MgmtFlavor: "remote-management", + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), + MgmtFlavor: "topology", InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.4/clusterctl-{OS}-{ARCH}", InitWithCoreProvider: "cluster-api:v1.5.4", InitWithBootstrapProviders: []string{"kubeadm:v1.5.4"}, @@ -54,7 +57,8 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.7= BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - MgmtFlavor: "remote-management", + MgmtFlavor: "topology", + PreUpgrade: vsphereframework.LoadImagesFunc(ctx), InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.9/clusterctl-{OS}-{ARCH}", InitWithCoreProvider: "cluster-api:v1.4.9", InitWithBootstrapProviders: []string{"kubeadm:v1.4.9"}, diff --git a/test/e2e/config/vsphere-ci.yaml b/test/e2e/config/vsphere-ci.yaml index 300561d481..2d4a4b783d 100644 --- a/test/e2e/config/vsphere-ci.yaml +++ b/test/e2e/config/vsphere-ci.yaml @@ -151,7 +151,6 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-node-drain.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ownerreferences.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-pci.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-remote-management.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-storage-policy.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-topology.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template.yaml" diff --git a/test/e2e/config/vsphere-dev.yaml b/test/e2e/config/vsphere-dev.yaml index ed7ce3fa1c..da0b2cf389 100644 --- a/test/e2e/config/vsphere-dev.yaml +++ b/test/e2e/config/vsphere-dev.yaml @@ -154,7 +154,6 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-node-drain.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ownerreferences.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-pci.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-remote-management.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-storage-policy.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-topology.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template.yaml" diff --git a/test/e2e/data/infrastructure-vsphere/main/remote-management/image-injection.yaml b/test/e2e/data/infrastructure-vsphere/main/remote-management/image-injection.yaml deleted file mode 100644 index 0cbda95f31..0000000000 --- a/test/e2e/data/infrastructure-vsphere/main/remote-management/image-injection.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- op: add - path: /spec/topology/variables/- - value: - name: preKubeadmScript - value: | - mkdir -p /opt/cluster-api - curl "https://storage.googleapis.com/capv-ci/${E2E_IMAGE_SHA}" -o /opt/cluster-api/image.tar - ctr -n k8s.io images import /opt/cluster-api/image.tar diff --git a/test/e2e/data/infrastructure-vsphere/main/remote-management/kustomization.yaml b/test/e2e/data/infrastructure-vsphere/main/remote-management/kustomization.yaml deleted file mode 100644 index a3096d07f7..0000000000 --- a/test/e2e/data/infrastructure-vsphere/main/remote-management/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../topology -patches: - - target: - kind: Cluster - path: ./image-injection.yaml diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go new file mode 100644 index 0000000000..c4f88cdc9d --- /dev/null +++ b/test/framework/daemonset_helpers.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// waitForDaemonSetAvailableInput is the input for waitForDaemonSetAvailable. +type waitForDaemonSetAvailableInput struct { + Getter framework.Getter + Daemonset *appsv1.DaemonSet +} + +// waitForDaemonSetAvailable waits until the DaemonSet is rolled out: +// * status.updatedNumberScheduled < status.DesiredNumberScheduled. +// * status.NumberAvailable < status.DesiredNumberScheduled. +func waitForDaemonSetAvailable(ctx context.Context, input waitForDaemonSetAvailableInput, intervals ...interface{}) { + Byf("Waiting for daemonset %s to be available", klog.KObj(input.Daemonset)) + daemonSet := &appsv1.DaemonSet{} + Eventually(func() bool { + key := client.ObjectKey{ + Namespace: input.Daemonset.GetNamespace(), + Name: input.Daemonset.GetName(), + } + if err := input.Getter.Get(ctx, key, daemonSet); err != nil { + return false + } + if daemonSet.Generation <= daemonSet.Status.ObservedGeneration { + if daemonSet.Status.UpdatedNumberScheduled < daemonSet.Status.DesiredNumberScheduled { + return false + } + if daemonSet.Status.NumberAvailable < daemonSet.Status.DesiredNumberScheduled { + return false + } + return true + } + return false + }, intervals...).Should(BeTrue()) +} diff --git a/test/framework/image_preloading.go b/test/framework/image_preloading.go new file mode 100644 index 0000000000..13eebe1de8 --- /dev/null +++ b/test/framework/image_preloading.go @@ -0,0 +1,219 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func LoadImagesFunc(ctx context.Context) func(clusterProxy framework.ClusterProxy) { + sourceFile := os.Getenv("DOCKER_IMAGE_TAR") + Expect(sourceFile).ToNot(BeEmpty(), "DOCKER_IMAGE_TAR must be set") + + return func(clusterProxy framework.ClusterProxy) { + loadImagesToCluster(ctx, sourceFile, clusterProxy) + } +} + +// loadImagesToCluster deploys a privileged daemonset and uses it to stream-load container images. +func loadImagesToCluster(ctx context.Context, sourceFile string, clusterProxy framework.ClusterProxy) { + daemonSet, daemonSetMutateFn, daemonSetLabels := getPreloadDaemonset() + ctrlClient := clusterProxy.GetClient() + + // Create the DaemonSet. + _, err := controllerutil.CreateOrPatch(ctx, ctrlClient, daemonSet, daemonSetMutateFn) + Expect(err).ToNot(HaveOccurred()) + + // Wait for DaemonSet to be available. + waitForDaemonSetAvailable(ctx, waitForDaemonSetAvailableInput{Getter: ctrlClient, Daemonset: daemonSet}, time.Minute*3, time.Second*10) + + // List all pods and load images via each found pod. + pods := &corev1.PodList{} + Expect(ctrlClient.List( + ctx, + pods, + client.InNamespace(daemonSet.Namespace), + client.MatchingLabels(daemonSetLabels), + )).To(Succeed()) + + errs := []error{} + for j := range pods.Items { + pod := pods.Items[j] + Byf("Loading images to node %s via pod %s", pod.Spec.NodeName, klog.KObj(&pod)) + if err := loadImagesViaPod(ctx, clusterProxy, sourceFile, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name); err != nil { + errs = append(errs, err) + } + } + Expect(kerrors.NewAggregate(errs)).ToNot(HaveOccurred()) + + // Delete the DaemonSet. + Expect(ctrlClient.Delete(ctx, daemonSet)).To(Succeed()) +} + +func loadImagesViaPod(ctx context.Context, clusterProxy framework.ClusterProxy, sourceFile, namespace, podName, containerName string) error { + // Open source tar file. + reader, writer := io.Pipe() + file, err := os.Open(filepath.Clean(sourceFile)) + if err != nil { + return err + } + + // Use go routine to pipe source file content into then stdin. + go func(file *os.File, writer io.WriteCloser) { + defer writer.Close() + defer file.Close() + // Ignoring the error here because the execPod command should fail in case of + // failure copying over the data. + _, err := io.Copy(writer, file) + if err != nil { + fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to copy file data to io.Pipe: %v\n", err) + } + }(file, writer) + + // Load the container images using ctr and delete the file. + loadCommand := "ctr -n k8s.io images import -" + return execPod(ctx, clusterProxy, namespace, podName, containerName, loadCommand, reader) +} + +// execPod executes a command at a pod. +// xref: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubectl/pkg/cmd/exec/exec.go#L123 +func execPod(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, podName, containerName, cmd string, stdin io.Reader) error { + var hasStdin bool + if stdin != nil { + hasStdin = true + } + + req := clusterProxy.GetClientSet().CoreV1().RESTClient().Post(). + Namespace(namespace). + Resource("pods"). + Name(podName). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: []string{"/bin/sh", "-c", cmd}, + Stdin: hasStdin, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(clusterProxy.GetRESTConfig(), "POST", req.URL()) + if err != nil { + return err + } + + var stdout, stderr bytes.Buffer + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: &stdout, + Stderr: &stderr, + Tty: false, + }) + if err != nil { + return errors.Wrapf(err, "running command %q stdout=%q, stderr=%q", cmd, stdout.String(), stderr.String()) + } + + return nil +} + +func getPreloadDaemonset() (*appsv1.DaemonSet, controllerutil.MutateFn, map[string]string) { + labels := map[string]string{ + "app": "image-preloader", + } + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: "image-preloader", + Labels: labels, + }, + } + mutateFunc := func() error { + ds.Labels = labels + ds.Spec = appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "pause", + Image: "registry.k8s.io/pause:3.9", + Command: []string{"/usr/bin/tail", "-f", "/dev/null"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host", + MountPath: "/", + }, + }, + }, + }, + HostPID: true, + HostIPC: true, + Volumes: []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + Type: ptr.To(corev1.HostPathDirectory), + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + // Tolerate any taint. + { + Operator: corev1.TolerationOpExists, + }, + }, + }, + }, + } + return nil + } + return ds, mutateFunc, labels +} diff --git a/test/go.mod b/test/go.mod index f4f2b821c4..c00f1aacf0 100644 --- a/test/go.mod +++ b/test/go.mod @@ -9,7 +9,7 @@ replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.6.1 replace sigs.k8s.io/cluster-api-provider-vsphere => ../ require ( - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.14.0 github.com/onsi/gomega v1.30.0 github.com/pkg/errors v0.9.1 @@ -87,6 +87,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/test/go.sum b/test/go.sum index 584f7384f0..3d131dfc83 100644 --- a/test/go.sum +++ b/test/go.sum @@ -83,6 +83,7 @@ github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/g github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -312,8 +313,8 @@ github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -436,6 +437,7 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=