diff --git a/hack/ci-e2e-lib.sh b/hack/ci-e2e-lib.sh new file mode 100644 index 0000000000..2bf32ffc8d --- /dev/null +++ b/hack/ci-e2e-lib.sh @@ -0,0 +1,185 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Note: This is a copy of cluster-api's scripts/ci-e2e-lib.sh. + +# k8s::prepareKindestImagesVariables defaults the environment variable KUBERNETES_VERSION_MANAGEMENT +# depending on what is set in GINKGO_FOCUS. +# Note: We do this to ensure that the kindest/node image gets built if it does +# not already exist, e.g. for pre-releases, but only if necessary. +k8s::prepareKindestImagesVariables() { + # Always default KUBERNETES_VERSION_MANAGEMENT because we always create a management cluster out of it. + if [[ -z "${KUBERNETES_VERSION_MANAGEMENT:-}" ]]; then + KUBERNETES_VERSION_MANAGEMENT=$(grep KUBERNETES_VERSION_MANAGEMENT: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') + echo "Defaulting KUBERNETES_VERSION_MANAGEMENT to ${KUBERNETES_VERSION_MANAGEMENT} to trigger image build (because env var is not set)" + fi +} + +# k8s::prepareKindestImages checks all the e2e test variables representing a Kubernetes version, +# and makes sure a corresponding kindest/node image is available locally. +k8s::prepareKindestImages() { + if [ -n "${KUBERNETES_VERSION_MANAGEMENT:-}" ]; then + k8s::resolveVersion "KUBERNETES_VERSION_MANAGEMENT" "$KUBERNETES_VERSION_MANAGEMENT" + export KUBERNETES_VERSION_MANAGEMENT=$resolveVersion + + kind::prepareKindestImage "$resolveVersion" + fi +} + +# k8s::resolveVersion resolves kubernetes version labels (e.g. latest) to the corresponding version numbers. +# The result will be available in the resolveVersion variable which is accessible from the caller. +# +# NOTE: this can't be used for kindest/node images pulled from docker hub, given that there are not guarantees that +# such images are generated in sync with the Kubernetes release process. +k8s::resolveVersion() { + local variableName=$1 + local version=$2 + + resolveVersion=$version + if [[ "$version" =~ ^v ]]; then + echo "+ $variableName=\"$version\" already a version, no need to resolve" + return + fi + + if [[ "$version" =~ ^ci/ ]]; then + resolveVersion=$(curl -LsS "http://dl.k8s.io/ci/${version#ci/}.txt") + else + resolveVersion=$(curl -LsS "http://dl.k8s.io/release/${version}.txt") + fi + echo "+ $variableName=\"$version\" resolved to \"$resolveVersion\"" +} + +# kind::prepareKindestImage check if a kindest/image exist, and if yes, pre-pull it; otherwise it builds +# the kindest image locally +kind::prepareKindestImage() { + local version=$1 + + # ALWAYS_BUILD_KIND_IMAGES will default to false if unset. + ALWAYS_BUILD_KIND_IMAGES="${ALWAYS_BUILD_KIND_IMAGES:-"false"}" + + # Note: this is needed compared to CAPI due to `set -o nounset` in the parent script. + retVal=0 + + # Try to pre-pull the image + kind::prepullImage "kindest/node:$version" + + # if pre-pull failed, or ALWAYS_BUILD_KIND_IMAGES is true build the images locally. + if [[ "$retVal" != 0 ]] || [[ "$ALWAYS_BUILD_KIND_IMAGES" = "true" ]]; then + echo "+ building image for Kuberentes $version locally. This is either because the image wasn't available in docker hub or ALWAYS_BUILD_KIND_IMAGES is set to true" + kind::buildNodeImage "$version" + fi +} + +# kind::buildNodeImage builds a kindest/node images starting from Kubernetes sources. +# the func expect an input parameter defining the image tag to be used. +kind::buildNodeImage() { + local version=$1 + + # move to the Kubernetes repository. + echo "KUBE_ROOT $GOPATH/src/k8s.io/kubernetes" + cd "$GOPATH/src/k8s.io/kubernetes" || exit + + # checkouts the Kubernetes branch for the given version. + k8s::checkoutBranch "$version" + + # sets the build version that will be applied by the Kubernetes build command called during kind build node-image. + k8s::setBuildVersion "$version" + + # build the node image + version="${version//+/_}" + echo "+ Building kindest/node:$version" + kind build node-image --image "kindest/node:$version" + + # move back to Cluster API + cd "$REPO_ROOT" || exit +} + +# k8s::checkoutBranch checkouts the Kubernetes branch for the given version. +k8s::checkoutBranch() { + local version=$1 + echo "+ Checkout branch for Kubernetes $version" + + # checkout the required tag/branch. + local buildMetadata + buildMetadata=$(echo "${version#v}" | awk '{split($0,a,"+"); print a[2]}') + if [[ "$buildMetadata" == "" ]]; then + # if there are no release metadata, it means we are looking for a Kubernetes version that + # should be already been tagged. + echo "+ checkout tag $version" + git fetch --all --tags + git checkout "tags/$version" -B "$version-branch" + else + # otherwise we are requiring a Kubernetes version that should be built from HEAD + # of one of the existing branches + echo "+ checking for existing branches" + git fetch --all + + local major + local minor + major=$(echo "${version#v}" | awk '{split($0,a,"."); print a[1]}') + minor=$(echo "${version#v}" | awk '{split($0,a,"."); print a[2]}') + + local releaseBranch + releaseBranch="$(git branch -r | grep "release-$major.$minor$" || true)" + if [[ "$releaseBranch" != "" ]]; then + # if there is already a release branch for the required Kubernetes branch, use it + echo "+ checkout $releaseBranch branch" + git checkout "$releaseBranch" -B "release-$major.$minor" + else + # otherwise, we should build from the main branch, which is the branch for the next release + # TODO(sbueringer) we can only change this to main after k/k has migrated to main + echo "+ checkout master branch" + git checkout master + fi + fi +} + +# k8s::setBuildVersion sets the build version that will be applied by the Kubernetes build command. +# the func expect an input parameter defining the version to be used. +k8s::setBuildVersion() { + local version=$1 + echo "+ Setting version for Kubernetes build to $version" + + local major + local minor + major=$(echo "${version#v}" | awk '{split($0,a,"."); print a[1]}') + minor=$(echo "${version#v}" | awk '{split($0,a,"."); print a[2]}') + + cat > build-version << EOL +export KUBE_GIT_MAJOR=$major +export KUBE_GIT_MINOR=$minor +export KUBE_GIT_VERSION=$version +export KUBE_GIT_TREE_STATE=clean +export KUBE_GIT_COMMIT=d34db33f +EOL + + export KUBE_GIT_VERSION_FILE=$PWD/build-version +} + +# kind:prepullImage pre-pull a docker image if no already present locally. +# The result will be available in the retVal value which is accessible from the caller. +kind::prepullImage () { + local image=$1 + image="${image//+/_}" + + retVal=0 + if [[ "$(docker images -q "$image" 2> /dev/null)" == "" ]]; then + echo "+ Pulling $image" + docker pull "$image" || retVal=$? + else + echo "+ image $image already present in the system, skipping pre-pull" + fi +} diff --git a/hack/e2e.sh b/hack/e2e.sh index 85b9cc12eb..c0fa08e6a5 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -21,6 +21,9 @@ set -o pipefail # any non-zero exit code in a piped command causes the pipeline export PATH=${PWD}/hack/tools/bin:${PATH} REPO_ROOT=$(git rev-parse --show-toplevel) +# shellcheck source=./hack/ci-e2e-lib.sh +source "${REPO_ROOT}/hack/ci-e2e-lib.sh" + RE_VCSIM='\[vcsim\\]' # In CI, ARTIFACTS is set to a different directory. This stores the value of @@ -162,21 +165,18 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then fi fi -make envsubst +make envsubst kind + +# Prepare kindest/node images for all the required Kubernetes version; this implies +# 1. Kubernetes version labels (e.g. latest) to the corresponding version numbers. +# 2. Pre-pulling the corresponding kindest/node image if available; if not, building the image locally. +# Following variables are currently checked (if defined): +# - KUBERNETES_VERSION_MANAGEMENT +k8s::prepareKindestImagesVariables +k8s::prepareKindestImages # Only pre-pull vm-operator image where running in supervisor mode. if [[ ${GINKGO_FOCUS:-} =~ \\\[supervisor\\\] ]]; then - kind::prepullImage () { - local image=$1 - image="${image//+/_}" - - if [[ "$(docker images -q "$image" 2> /dev/null)" == "" ]]; then - echo "+ Pulling $image" - docker pull "$image" - else - echo "+ image $image already present in the system, skipping pre-pull" - fi - } kind::prepullImage "gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:${E2E_VM_OPERATOR_VERSION}" fi diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index c169ec60e6..ad695d68d7 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -267,6 +267,7 @@ providers: variables: # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml + KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.0" KUBERNETES_VERSION: "v1.30.0" KUBERNETES_VERSION_UPGRADE_FROM: "v1.29.0" KUBERNETES_VERSION_UPGRADE_TO: "v1.30.0" diff --git a/test/framework/framework.go b/test/framework/framework.go index 8e8b9eb327..9d24f0a375 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -191,6 +191,7 @@ func SetupBootstrapCluster(ctx context.Context, config *clusterctl.E2EConfig, sc clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ Name: config.ManagementClusterName, RequiresDockerSock: config.HasDockerProvider(), + KubernetesVersion: config.GetVariable(capi_e2e.KubernetesVersionManagement), Images: config.Images, })