From 22e375976460683847f67087c39b3fd6e35ce276 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Tue, 6 Aug 2024 10:00:34 +0200 Subject: [PATCH] test: inject bootstrap script for all templates --- .../main/base/kustomization.yaml | 6 + .../patch-k8s-install-script-kcp.yaml} | 14 +- .../base/patch-k8s-install-script-kct.yaml} | 17 +- .../main/clusterclass/kustomization.yaml | 3 + .../patch-k8s-install-script.yaml | 241 ++++++++++++++++++ .../install-on-bootstrap/kustomization.yaml | 3 - .../main/base/kustomization.yaml | 7 + .../base/patch-k8s-install-script-kcp.yaml | 109 ++++++++ .../base/patch-k8s-install-script-kct.yaml | 112 ++++++++ .../main/clusterclass/kustomization.yaml | 3 + .../patch-k8s-install-script.yaml | 241 ++++++++++++++++++ .../install-on-bootstrap/kustomization.yaml | 3 - 12 files changed, 739 insertions(+), 20 deletions(-) rename test/e2e/data/infrastructure-vsphere-govmomi/main/{install-on-bootstrap/inject-install-on-bootstrap.yaml => base/patch-k8s-install-script-kcp.yaml} (92%) rename test/e2e/data/{infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml => infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml} (92%) create mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kcp.yaml create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kct.yaml create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-k8s-install-script.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml index 2ef370ebd3..49573b877c 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml @@ -12,3 +12,9 @@ patches: - target: kind: VSphereMachineTemplate path: ../commons/remove-storage-policy.yaml + - target: + kind: KubeadmControlPlane + path: ./patch-k8s-install-script-kcp.yaml + - target: + kind: KubeadmConfigTemplate + path: ./patch-k8s-install-script-kct.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml similarity index 92% rename from test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml index 6abed91df8..060b49613c 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml @@ -1,10 +1,10 @@ -# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl - op: add - path: /spec/topology/variables/- + path: /spec/kubeadmConfigSpec/files/- value: - name: preKubeadmScript - # Note: This file has to be escaped as it is run through envsubst by clusterctl - value: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | #!/bin/bash set -o nounset @@ -36,8 +36,8 @@ LINE_SEPARATOR="*************************************************" echo "$${LINE_SEPARATOR}" - ## Variables (replaced by JSON patching) - KUBERNETES_VERSION=$${KUBERNETES_VERSION} + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} ## # Note: We assume if kubectl has the right version, everything else has as well diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml similarity index 92% rename from test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml index 6abed91df8..4ebf30cc17 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml @@ -1,10 +1,13 @@ -# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl - op: add - path: /spec/topology/variables/- + path: /spec/template/spec/files + value: [] +- op: add + path: /spec/template/spec/files/- value: - name: preKubeadmScript - # Note: This file has to be escaped as it is run through envsubst by clusterctl - value: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | #!/bin/bash set -o nounset @@ -36,8 +39,8 @@ LINE_SEPARATOR="*************************************************" echo "$${LINE_SEPARATOR}" - ## Variables (replaced by JSON patching) - KUBERNETES_VERSION=$${KUBERNETES_VERSION} + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} ## # Note: We assume if kubectl has the right version, everything else has as well diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml index 4c0e41b050..4128896681 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml @@ -9,6 +9,9 @@ patches: - target: kind: ClusterClass path: ./patch-prekubeadmscript.yaml + - target: + kind: ClusterClass + path: ./patch-k8s-install-script.yaml - target: kind: ClusterClass path: ./patch-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml new file mode 100644 index 0000000000..e04d690a6f --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml @@ -0,0 +1,241 @@ +- op: add + path: /spec/patches/- + value: + definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.controlPlane.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.machineDeployment.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: k8sInstallScript diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/kustomization.yaml index 6baeb19c74..335d205b7d 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/kustomization.yaml @@ -3,9 +3,6 @@ kind: Kustomization resources: - ../topology patches: - - target: - kind: Cluster - path: ./inject-install-on-bootstrap.yaml - target: kind: Cluster path: ./set-md-rollout-strategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml index 08f9abeccc..9907ca0906 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml @@ -6,3 +6,10 @@ resources: patchesStrategicMerge: - ../commons/cluster-resource-set-label.yaml - ../commons/cluster-network-CIDR.yaml +patches: + - target: + kind: KubeadmControlPlane + path: ./patch-k8s-install-script-kcp.yaml + - target: + kind: KubeadmConfigTemplate + path: ./patch-k8s-install-script-kct.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kcp.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kcp.yaml new file mode 100644 index 0000000000..060b49613c --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kcp.yaml @@ -0,0 +1,109 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kct.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kct.yaml new file mode 100644 index 0000000000..4ebf30cc17 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kct.yaml @@ -0,0 +1,112 @@ +- op: add + path: /spec/template/spec/files + value: [] +- op: add + path: /spec/template/spec/files/- + value: + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml index a0dabb0005..6858997f52 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml @@ -9,6 +9,9 @@ patches: - target: kind: ClusterClass path: ./patch-prekubeadmscript.yaml + - target: + kind: ClusterClass + path: ./patch-k8s-install-script.yaml - target: kind: ClusterClass path: ./patch-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-k8s-install-script.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-k8s-install-script.yaml new file mode 100644 index 0000000000..e04d690a6f --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-k8s-install-script.yaml @@ -0,0 +1,241 @@ +- op: add + path: /spec/patches/- + value: + definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.controlPlane.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.machineDeployment.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: k8sInstallScript diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml index 6baeb19c74..335d205b7d 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml @@ -3,9 +3,6 @@ kind: Kustomization resources: - ../topology patches: - - target: - kind: Cluster - path: ./inject-install-on-bootstrap.yaml - target: kind: Cluster path: ./set-md-rollout-strategy.yaml