Skip to content

Commit

Permalink
Merge pull request #1269 from AartiJivrajani/v1beta1-cluster-update-test
Browse files Browse the repository at this point in the history
add v1alpha3/v1alpha4->v1beta1 clusterctl upgrade test
  • Loading branch information
k8s-ci-robot authored Oct 19, 2021
2 parents d0febf0 + 91a2b5f commit 1ebdff7
Show file tree
Hide file tree
Showing 15 changed files with 1,990 additions and 36 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
# Generated files
/cluster-api-provider-vsphere
/examples/provider-components/provider-components*.yaml
test/e2e/data/infrastructure-vsphere/kustomization/base/cluster-template.yaml
test/e2e/data/infrastructure-vsphere/cluster-template.yaml
test/e2e/data/infrastructure-vsphere/cluster-template-remote-management.yaml

# env vars file used in getting-started.md and manifests generation
envvars.txt
Expand Down
24 changes: 19 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -119,20 +119,34 @@ test: $(GOVC)
e2e-image: ## Build the e2e manager image
docker build --build-arg ldflags="$(LDFLAGS)" --tag="gcr.io/k8s-staging-cluster-api/capv-manager:e2e" .

.PHONY: e2e-templates
e2e-templates: ## Generate e2e cluster templates
$(MAKE) release-manifests
cp $(RELEASE_DIR)/cluster-template.yaml $(E2E_TEMPLATE_DIR)/kustomization/base/cluster-template.yaml
"$(KUSTOMIZE)" build $(E2E_TEMPLATE_DIR)/kustomization/base > $(E2E_TEMPLATE_DIR)/cluster-template.yaml
"$(KUSTOMIZE)" build $(E2E_TEMPLATE_DIR)/kustomization/remote-management > $(E2E_TEMPLATE_DIR)/cluster-template-remote-management.yaml

.PHONY: e2e
e2e: e2e-image
e2e: e2e-image e2e-templates
e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
$(MAKE) release-manifests
cp $(RELEASE_DIR)/cluster-template.yaml $(E2E_TEMPLATE_DIR)/kustomization/cluster-template.yaml
"$(KUSTOMIZE)" build $(E2E_TEMPLATE_DIR)/kustomization > $(E2E_TEMPLATE_DIR)/cluster-template.yaml
@echo PATH=$(PATH)
@echo
@echo Contents of $(TOOLS_BIN_DIR):
@ls $(TOOLS_BIN_DIR)
@echo

time $(GINKGO) -v ./test/e2e -- --e2e.config="$(E2E_CONF_FILE)"
time $(GINKGO) -v -skip="ClusterAPI Upgrade Tests" ./test/e2e -- --e2e.config="$(E2E_CONF_FILE)"

.PHONY: e2e-upgrade
e2e-upgrade: e2e-image e2e-templates
e2e-upgrade: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
@echo PATH=$(PATH)
@echo
@echo Contents of $(TOOLS_BIN_DIR):
@ls $(TOOLS_BIN_DIR)
@echo

time $(GINKGO) -v -focus="ClusterAPI Upgrade Tests" ./test/e2e -- --e2e.config="$(E2E_CONF_FILE)"
## --------------------------------------
## Binaries
## --------------------------------------
Expand Down
45 changes: 42 additions & 3 deletions hack/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,23 +26,40 @@ source "${REPO_ROOT}/hack/ensure-kubectl.sh"

on_exit() {
# release IPClaim
echo "Releasing IP claim"
echo "Releasing IP claims"
kubectl --kubeconfig="${KUBECONFIG}" delete ipclaim "${IPCLAIM_NAME}" || true
kubectl --kubeconfig="${KUBECONFIG}" delete ipclaim "${WORKLOAD_IPCLAIM_NAME}" || true

# kill the VPN
docker kill vpn

# logout of gcloud
if [ "${AUTH}" ]; then
gcloud auth revoke
fi
}

trap on_exit EXIT

function login() {
# If GCR_KEY_FILE is set, use that service account to login
if [ "${GCR_KEY_FILE}" ]; then
gcloud auth activate-service-account --key-file "${GCR_KEY_FILE}" || fatal "unable to login"
AUTH=1
fi
}

AUTH=
E2E_IMAGE_SHA=
GCR_KEY_FILE="${GCR_KEY_FILE:-}"
export VSPHERE_SERVER="${GOVC_URL}"
export VSPHERE_USERNAME="${GOVC_USERNAME}"
export VSPHERE_PASSWORD="${GOVC_PASSWORD}"
export VSPHERE_SSH_AUTHORIZED_KEY="${VM_SSH_PUB_KEY}"
export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key"
export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere-ci.yaml"
export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"

export DOCKER_IMAGE_TAR="${ARTIFACTS}/tempContainers/image.tar"
export GC_KIND="false"

# Run the vpn client in container
Expand All @@ -64,8 +81,30 @@ sed "s/IPCLAIM_NAME/${IPCLAIM_NAME}/" "${REPO_ROOT}/hack/ipclaim-template.yaml"
IPADDRESS_NAME=$(kubectl --kubeconfig=${KUBECONFIG} get ipclaim "${IPCLAIM_NAME}" -o=jsonpath='{@.status.address.name}')
CONTROL_PLANE_ENDPOINT_IP=$(kubectl --kubeconfig=${KUBECONFIG} get ipaddresses "${IPADDRESS_NAME}" -o=jsonpath='{@.spec.address}')
export CONTROL_PLANE_ENDPOINT_IP

echo "Acquired Control Plane IP: $CONTROL_PLANE_ENDPOINT_IP"

# Retrieve an IP to be used for the workload cluster in v1a3/v1a4 -> v1b1 upgrade tests
WORKLOAD_IPCLAIM_NAME="workload-ip-claim-$(date +%s)"
sed "s/IPCLAIM_NAME/${WORKLOAD_IPCLAIM_NAME}/" "${REPO_ROOT}/hack/ipclaim-template.yaml" | kubectl --kubeconfig=${KUBECONFIG} create -f -
WORKLOAD_IPADDRESS_NAME=$(kubectl --kubeconfig=${KUBECONFIG} get ipclaim "${WORKLOAD_IPCLAIM_NAME}" -o=jsonpath='{@.status.address.name}')
WORKLOAD_CONTROL_PLANE_ENDPOINT_IP=$(kubectl --kubeconfig=${KUBECONFIG} get ipaddresses "${WORKLOAD_IPADDRESS_NAME}" -o=jsonpath='{@.spec.address}')
export WORKLOAD_CONTROL_PLANE_ENDPOINT_IP
echo "Acquired Workload Cluster Control Plane IP: $WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"

# save the docker image locally
make e2e-image
mkdir -p "$ARTIFACTS"/tempContainers
docker save gcr.io/k8s-staging-cluster-api/capv-manager:e2e -o "$DOCKER_IMAGE_TAR"

# store the image on gcs
login
E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' gcr.io/k8s-staging-cluster-api/capv-manager:e2e)
export E2E_IMAGE_SHA
gsutil cp "$ARTIFACTS"/tempContainers/image.tar gs://capv-ci/"$E2E_IMAGE_SHA"

# Run e2e-upgrade tests
# TODO: move e2e-upgrade into its own prow job
make e2e-upgrade

# Run e2e tests
make e2e
57 changes: 57 additions & 0 deletions test/e2e/capi_upgrade_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"

. "github.com/onsi/ginkgo"

capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
)

var _ = Context("ClusterAPI Upgrade Tests", func() {
Describe("Upgrading cluster from v1alpha3 to v1beta1 using clusterctl", func() {
capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: e2eConfig.GetVariable("INIT_WITH_BINARY_V1ALPHA3"),
InitWithProvidersContract: "v1alpha3",
MgmtFlavor: "remote-management",
}
})
})

Describe("Upgrading cluster from v1alpha4 to v1beta1 using clusterctl", func() {
capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: e2eConfig.GetVariable("INIT_WITH_BINARY_V1ALPHA4"),
InitWithProvidersContract: "v1alpha4",
MgmtFlavor: "remote-management",
}
})
})
})
92 changes: 79 additions & 13 deletions test/e2e/config/vsphere-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,24 @@ providers:
- name: cluster-api
type: CoreProvider
versions:
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/core-components.yaml
type: url
contract: v1alpha3
files:
- sourcePath: "../data/shared/metadata.yaml"
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
contract: v1alpha4
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/core-components.yaml"
type: "url"
files:
- sourcePath: "../data/shared/metadata.yaml"
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- name: v1.0.0
# Use manifest from source files
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/core-components.yaml"
Expand All @@ -37,12 +55,28 @@ providers:
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"

- name: kubeadm
type: BootstrapProvider
versions:
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/bootstrap-components.yaml
type: url
contract: v1alpha3
files:
- sourcePath: "../data/shared/metadata.yaml"
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/bootstrap-components.yaml"
type: "url"
contract: v1alpha4
files:
- sourcePath: "../data/shared/metadata.yaml"
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- name: v1.0.0
# Use manifest from source files
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/bootstrap-components.yaml"
Expand All @@ -52,12 +86,28 @@ providers:
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"

- name: kubeadm
type: ControlPlaneProvider
versions:
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/control-plane-components.yaml
type: url
contract: v1alpha3
files:
- sourcePath: "../data/shared/metadata.yaml"
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- name: v0.4.4
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/control-plane-components.yaml"
type: "url"
contract: v1alpha4
files:
- sourcePath: "../data/shared/metadata.yaml"
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- name: v1.0.0
# Use manifest from source files
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/control-plane-components.yaml"
Expand All @@ -67,25 +117,38 @@ providers:
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"

- name: vsphere
type: InfrastructureProvider
versions:
- name: v0.7.10
value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v0.7.10/infrastructure-components.yaml
type: "url"
contract: v1alpha3
files:
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/capi-upgrades/v1alpha3/cluster-template.yaml"
- sourcePath: "../../../metadata.yaml"
- name: v0.8.1
value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v0.8.1/infrastructure-components.yaml
type: "url"
contract: v1alpha4
files:
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/capi-upgrades/v1alpha4/cluster-template.yaml"
- sourcePath: "../../../metadata.yaml"
- name: v1.0.0
# Use manifest from source files
value: ../../../../cluster-api-provider-vsphere/config/default
contract: v1beta1
replacements:
- old: gcr.io/cluster-api-provider-vsphere/release/manager:latest
new: gcr.io/k8s-staging-cluster-api/capv-manager:e2e
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"

files:
# Add a cluster template
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template.yaml"
- sourcePath: "../../../metadata.yaml"
files:
# Add a cluster template
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-remote-management.yaml"
- sourcePath: "../../../metadata.yaml"

variables:
KUBERNETES_VERSION: "v1.19.1"
Expand All @@ -102,11 +165,14 @@ variables:
VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy"
VSPHERE_NETWORK: "sddc-cgw-network-6"
VSPHERE_TEMPLATE: "ubuntu-1804-kube-v1.19.1"
INIT_WITH_BINARY_V1ALPHA3: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
INIT_WITH_BINARY_V1ALPHA4: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}"
INIT_WITH_KUBERNETES_VERSION: "v1.19.1"

intervals:
default/wait-controllers: ["5m", "10s"]
default/wait-cluster: ["5m", "10s"]
default/wait-control-plane: ["5m", "10s"]
default/wait-worker-nodes: ["5m", "10s"]
default/wait-control-plane: ["10m", "10s"]
default/wait-worker-nodes: ["10m", "10s"]
default/wait-delete-cluster: ["5m", "10s"]
default/wait-machine-upgrade: ["15m", "1m"]
Loading

0 comments on commit 1ebdff7

Please sign in to comment.