-
Notifications
You must be signed in to change notification settings - Fork 295
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test: inject bootstrap script for all templates
- Loading branch information
Showing
12 changed files
with
739 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
241 changes: 241 additions & 0 deletions
241
test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,241 @@ | ||
| - op: add | ||
| path: /spec/patches/- | ||
| value: | ||
| definitions: | ||
| - jsonPatches: | ||
| - op: add | ||
| path: /spec/template/spec/kubeadmConfigSpec/files/- | ||
| valueFrom: | ||
| template: | | ||
| owner: root:root | ||
| path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" | ||
| permissions: "0755" | ||
| content: | | ||
| #!/bin/bash | ||
| set -o nounset | ||
| set -o pipefail | ||
| set -o errexit | ||
| function retry { | ||
| attempt=0 | ||
| max_attempts=$${1} | ||
| interval=$${2} | ||
| shift; shift | ||
| until [[ $${attempt} -ge "$${max_attempts}" ]] ; do | ||
| attempt=$((attempt+1)) | ||
| set +e | ||
| eval "$*" && return || echo "failed $${attempt} times: $*" | ||
| set -e | ||
| sleep "$${interval}" | ||
| done | ||
| echo "error: reached max attempts at retry($*)" | ||
| return 1 | ||
| } | ||
| [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" | ||
| # This test installs release packages or binaries that are a result of the CI and release builds. | ||
| # It runs '... --version' commands to verify that the binaries are correctly installed | ||
| # and finally uninstalls the packages. | ||
| # For the release packages it tests all versions in the support skew. | ||
| LINE_SEPARATOR="*************************************************" | ||
| echo "$${LINE_SEPARATOR}" | ||
| ## Variables (replaced by JSON patching) | ||
| KUBERNETES_VERSION={{ .builtin.controlPlane.version }} | ||
| ## | ||
| # Note: We assume if kubectl has the right version, everything else has as well | ||
| if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then | ||
| echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" | ||
| exit 0 | ||
| fi | ||
| if [[ "$${KUBERNETES_VERSION}" != "" ]]; then | ||
| CI_DIR=/tmp/k8s-ci | ||
| mkdir -p "$${CI_DIR}" | ||
| declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") | ||
| # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. | ||
| declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") | ||
| CONTAINER_EXT="tar" | ||
| echo "* testing CI version $${KUBERNETES_VERSION}" | ||
| # Check for semver | ||
| if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | ||
| CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" | ||
| VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" | ||
| export DEBIAN_FRONTEND=noninteractive | ||
| # sometimes the network is not immediately available, so we have to retry the apt-get update | ||
| retry 10 5 "apt-get update" | ||
| apt-get install -y apt-transport-https ca-certificates curl gpg | ||
| mkdir -p -m 755 /etc/apt/keyrings | ||
| curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg | ||
| echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list | ||
| apt-get update | ||
| # replace . with \. | ||
| VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" | ||
| PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" | ||
| for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do | ||
| echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" | ||
| apt-mark unhold "$${CI_PACKAGE}" | ||
| apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" | ||
| apt-mark hold "$${CI_PACKAGE}" | ||
| done | ||
| systemctl restart kubelet | ||
| else | ||
| CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" | ||
| for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do | ||
| # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev | ||
| # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl | ||
| echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" | ||
| wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" | ||
| chmod +x "$${CI_DIR}/$${CI_PACKAGE}" | ||
| mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" | ||
| done | ||
| systemctl restart kubelet | ||
| fi | ||
| IMAGE_REGISTRY_PREFIX=registry.k8s.io | ||
| # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io | ||
| if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then | ||
| IMAGE_REGISTRY_PREFIX=k8s.gcr.io | ||
| fi | ||
| for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do | ||
| echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" | ||
| wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" | ||
| $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" | ||
| $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" | ||
| $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" | ||
| done | ||
| fi | ||
| echo "* checking binary versions" | ||
| echo "ctr version: " "$(ctr version)" | ||
| echo "kubeadm version: " "$(kubeadm version -o=short)" | ||
| echo "kubectl version: " "$(kubectl version --client=true)" | ||
| echo "kubelet version: " "$(kubelet --version)" | ||
| echo "$${LINE_SEPARATOR}" | ||
| selector: | ||
| apiVersion: controlplane.cluster.x-k8s.io/v1beta1 | ||
| kind: KubeadmControlPlaneTemplate | ||
| matchResources: | ||
| controlPlane: true | ||
| - jsonPatches: | ||
| - op: add | ||
| path: /spec/template/spec/files/- | ||
| valueFrom: | ||
| template: | | ||
| owner: root:root | ||
| path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" | ||
| permissions: "0755" | ||
| content: | | ||
| #!/bin/bash | ||
| set -o nounset | ||
| set -o pipefail | ||
| set -o errexit | ||
| function retry { | ||
| attempt=0 | ||
| max_attempts=$${1} | ||
| interval=$${2} | ||
| shift; shift | ||
| until [[ $${attempt} -ge "$${max_attempts}" ]] ; do | ||
| attempt=$((attempt+1)) | ||
| set +e | ||
| eval "$*" && return || echo "failed $${attempt} times: $*" | ||
| set -e | ||
| sleep "$${interval}" | ||
| done | ||
| echo "error: reached max attempts at retry($*)" | ||
| return 1 | ||
| } | ||
| [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" | ||
| # This test installs release packages or binaries that are a result of the CI and release builds. | ||
| # It runs '... --version' commands to verify that the binaries are correctly installed | ||
| # and finally uninstalls the packages. | ||
| # For the release packages it tests all versions in the support skew. | ||
| LINE_SEPARATOR="*************************************************" | ||
| echo "$${LINE_SEPARATOR}" | ||
| ## Variables (replaced by JSON patching) | ||
| KUBERNETES_VERSION={{ .builtin.machineDeployment.version }} | ||
| ## | ||
| # Note: We assume if kubectl has the right version, everything else has as well | ||
| if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then | ||
| echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" | ||
| exit 0 | ||
| fi | ||
| if [[ "$${KUBERNETES_VERSION}" != "" ]]; then | ||
| CI_DIR=/tmp/k8s-ci | ||
| mkdir -p "$${CI_DIR}" | ||
| declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") | ||
| # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. | ||
| declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") | ||
| CONTAINER_EXT="tar" | ||
| echo "* testing CI version $${KUBERNETES_VERSION}" | ||
| # Check for semver | ||
| if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | ||
| CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" | ||
| VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" | ||
| export DEBIAN_FRONTEND=noninteractive | ||
| # sometimes the network is not immediately available, so we have to retry the apt-get update | ||
| retry 10 5 "apt-get update" | ||
| apt-get install -y apt-transport-https ca-certificates curl gpg | ||
| mkdir -p -m 755 /etc/apt/keyrings | ||
| curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg | ||
| echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list | ||
| apt-get update | ||
| # replace . with \. | ||
| VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" | ||
| PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" | ||
| for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do | ||
| echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" | ||
| apt-mark unhold "$${CI_PACKAGE}" | ||
| apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" | ||
| apt-mark hold "$${CI_PACKAGE}" | ||
| done | ||
| systemctl restart kubelet | ||
| else | ||
| CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" | ||
| for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do | ||
| # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev | ||
| # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl | ||
| echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" | ||
| wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" | ||
| chmod +x "$${CI_DIR}/$${CI_PACKAGE}" | ||
| mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" | ||
| done | ||
| systemctl restart kubelet | ||
| fi | ||
| IMAGE_REGISTRY_PREFIX=registry.k8s.io | ||
| # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io | ||
| if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then | ||
| IMAGE_REGISTRY_PREFIX=k8s.gcr.io | ||
| fi | ||
| for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do | ||
| echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" | ||
| wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" | ||
| $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" | ||
| $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" | ||
| $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" | ||
| done | ||
| fi | ||
| echo "* checking binary versions" | ||
| echo "ctr version: " "$(ctr version)" | ||
| echo "kubeadm version: " "$(kubeadm version -o=short)" | ||
| echo "kubectl version: " "$(kubectl version --client=true)" | ||
| echo "kubelet version: " "$(kubelet --version)" | ||
| echo "$${LINE_SEPARATOR}" | ||
| selector: | ||
| apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 | ||
| kind: KubeadmConfigTemplate | ||
| matchResources: | ||
| machineDeploymentClass: | ||
| names: | ||
| - ${CLUSTER_CLASS_NAME}-worker | ||
| name: k8sInstallScript |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.