From 55b26bc7e0544f9c840c9d83a9ddaa74b869a3d8 Mon Sep 17 00:00:00 2001 From: Thomas Ferrandiz Date: Thu, 10 Aug 2023 09:13:57 +0000 Subject: [PATCH] Add chart validation tests validate-charts runs as part of 'make validate' step and checks that all images used in packaged charts: - use systemGlobalRegistry - are present in script/build-images Signed-off-by: Thomas Ferrandiz --- Dockerfile | 19 +----- Makefile | 7 +- Vagrantfile | 2 +- charts/build-charts.sh | 7 ++ charts/chart_versions.yaml | 46 +++++++++++++ scripts/build-images | 8 +-- scripts/validate | 2 +- scripts/validate-charts | 131 +++++++++++++++++++++++++++++++++++++ 8 files changed, 199 insertions(+), 23 deletions(-) create mode 100755 charts/build-charts.sh create mode 100644 charts/chart_versions.yaml create mode 100755 scripts/validate-charts diff --git a/Dockerfile b/Dockerfile index 939c0572b35..b36716439ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,8 @@ RUN set -x && \ py3-pip \ pigz \ tar \ - yq + yq \ + helm RUN if [ "${ARCH}" = "amd64" ]; then \ apk --no-cache add mingw-w64-gcc; \ @@ -105,21 +106,7 @@ ARG KUBERNETES_VERSION="" ARG CACHEBUST="cachebust" COPY charts/ /charts/ RUN echo ${CACHEBUST}>/dev/null -RUN CHART_VERSION="1.14.100" CHART_FILE=/charts/rke2-cilium.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="v3.26.1-build2023080200" CHART_FILE=/charts/rke2-canal.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="v3.26.101" CHART_FILE=/charts/rke2-calico.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="v3.26.101" CHART_FILE=/charts/rke2-calico-crd.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="1.24.004" CHART_FILE=/charts/rke2-coredns.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="4.6.100" CHART_FILE=/charts/rke2-ingress-nginx.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh -RUN CHART_VERSION="2.11.100-build2023051509" CHART_FILE=/charts/rke2-metrics-server.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh -RUN CHART_VERSION="v4.0.2-build2023081100" CHART_FILE=/charts/rke2-multus.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="1.5.100" CHART_FILE=/charts/rancher-vsphere-cpi.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="3.0.1-rancher101" CHART_FILE=/charts/rancher-vsphere-csi.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="0.2.200" CHART_FILE=/charts/harvester-cloud-provider.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="0.1.1600" CHART_FILE=/charts/harvester-csi-driver.yaml CHART_BOOTSTRAP=true /charts/build-chart.sh -RUN CHART_VERSION="1.7.202" CHART_FILE=/charts/rke2-snapshot-controller.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh -RUN CHART_VERSION="1.7.202" CHART_FILE=/charts/rke2-snapshot-controller-crd.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh -RUN CHART_VERSION="1.7.300" CHART_FILE=/charts/rke2-snapshot-validation-webhook.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh +RUN /charts/build-charts.sh RUN rm -vf /charts/*.sh /charts/*.md # rke2-runtime image diff --git a/Makefile b/Makefile index 335b790eef7..73444032fa0 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ ci-shell: clean .dapper ## Launch a shell in the CI environment .PHONY: dapper-ci dapper-ci: .ci ## Used by Drone CI, does the same as "ci" but in a Drone way -.ci: validate build package +.ci: validate validate-charts build package .PHONY: build build: ## Build using host go tools @@ -71,6 +71,11 @@ validate: ## Run go fmt/vet validate-release: ./scripts/validate-release +.PHONY: validate-charts +validate-charts: + ./scripts/validate-charts + + .PHONY: run run: build-debug ./scripts/run diff --git a/Vagrantfile b/Vagrantfile index e1b9febb2fe..6002a9f0d8f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -108,7 +108,7 @@ EOF # SELinux is Enforcing by default. # To set SELinux as Disabled on a VM that has already been provisioned: # SELINUX=Disabled vagrant up --provision-with=selinux - # To set SELinux as Permissive on a VM that has already been provsioned + # To set SELinux as Permissive on a VM that has already been provisioned # SELINUX=Permissive vagrant up --provision-with=selinux config.vm.provision "selinux", type: "shell", run: "once" do |sh| sh.upload_path = "/tmp/vagrant-selinux" diff --git a/charts/build-charts.sh b/charts/build-charts.sh new file mode 100755 index 00000000000..38dd2e62f58 --- /dev/null +++ b/charts/build-charts.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -eux -o pipefail + +while read version filename bootstrap; do + CHART_VERSION=$version CHART_FILE=$filename CHART_BOOTSTRAP=$bootstrap /charts/build-chart.sh +done <<< $(yq e '.charts[] | [.version, .filename, .bootstrap] | join(" ")' /charts/chart_versions.yaml) diff --git a/charts/chart_versions.yaml b/charts/chart_versions.yaml new file mode 100644 index 00000000000..d5954a7a035 --- /dev/null +++ b/charts/chart_versions.yaml @@ -0,0 +1,46 @@ +charts: + - version: 1.14.100 + filename: /charts/rke2-cilium.yaml + bootstrap: true + - version: v3.26.1-build2023080200 + filename: /charts/rke2-canal.yaml + bootstrap: true + - version: v3.26.101 + filename: /charts/rke2-calico.yaml + bootstrap: true + - version: v3.26.101 + filename: /charts/rke2-calico-crd.yaml + bootstrap: true + - version: 1.24.004 + filename: /charts/rke2-coredns.yaml + bootstrap: true + - version: 4.6.100 + filename: /charts/rke2-ingress-nginx.yaml + bootstrap: false + - version: 2.11.100-build2023051509 + filename: /charts/rke2-metrics-server.yaml + bootstrap: false + - version: v4.0.2-build2023081100 + filename: /charts/rke2-multus.yaml + bootstrap: true + - version: 1.5.100 + filename: /charts/rancher-vsphere-cpi.yaml + bootstrap: true + - version: 3.0.1-rancher101 + filename: /charts/rancher-vsphere-csi.yaml + bootstrap: true + - version: 0.2.200 + filename: /charts/harvester-cloud-provider.yaml + bootstrap: true + - version: 0.1.1600 + filename: /charts/harvester-csi-driver.yaml + bootstrap: true + - version: 1.7.202 + filename: /charts/rke2-snapshot-controller.yaml + bootstrap: false + - version: 1.7.202 + filename: /charts/rke2-snapshot-controller-crd.yaml + bootstrap: false + - version: 1.7.300 + filename: /charts/rke2-snapshot-validation-webhook.yaml + bootstrap: false diff --git a/scripts/build-images b/scripts/build-images index 6220eaee617..4df057f85c1 100755 --- a/scripts/build-images +++ b/scripts/build-images @@ -89,10 +89,10 @@ xargs -n1 -t docker image pull --quiet << EOF > build/images-harvester.txt ${REGISTRY}/rancher/harvester-cloud-provider:v0.2.0 ${REGISTRY}/rancher/mirrored-kube-vip-kube-vip-iptables:v0.6.0 ${REGISTRY}/rancher/harvester-csi-driver:v0.1.5 - ${REGISTRY}/rancher/longhornio-csi-node-driver-registrar:v2.3.0 - ${REGISTRY}/rancher/longhornio-csi-resizer:v1.2.0 - ${REGISTRY}/rancher/longhornio-csi-provisioner:v2.1.2 - ${REGISTRY}/rancher/longhornio-csi-attacher:v3.2.1 + ${REGISTRY}/rancher/mirrored-longhornio-csi-node-driver-registrar:v2.3.0 + ${REGISTRY}/rancher/mirrored-longhornio-csi-resizer:v1.2.0 + ${REGISTRY}/rancher/mirrored-longhornio-csi-provisioner:v2.1.2 + ${REGISTRY}/rancher/mirrored-longhornio-csi-attacher:v3.2.1 EOF fi # Continue to provide a legacy airgap archive set with the default CNI images diff --git a/scripts/validate b/scripts/validate index c4a84428368..5f46593c8b4 100755 --- a/scripts/validate +++ b/scripts/validate @@ -22,7 +22,7 @@ function check_win_binaries() { #fi CALICO_WINDOWS_VERSION=$(grep 'CALICO_VERSION=' Dockerfile.windows | cut -d '=' -f 2- | grep -oE "v([0-9]+)\.([0-9]+)") - CALICO_LINUX_VERSION=$(grep "rke2-calico.yaml" Dockerfile | grep 'CHART_VERSION=' | cut -d '=' -f 2- | grep -oE "v([0-9]+)\.([0-9]+)") + CALICO_LINUX_VERSION=$(yq '.charts[] | select(.filename == "/charts/rke2-canal.yaml").version' charts/chart_versions.yaml | cut -d ',' -f 1- | grep -oE "v([0-9]+)\.([0-9]+)") if [ ! "$CALICO_WINDOWS_VERSION" = "$CALICO_LINUX_VERSION" ]; then fatal "Calico windows binary version [$CALICO_WINDOWS_VERSION] does not match Calico chart version [$CALICO_LINUX_VERSION]" fi diff --git a/scripts/validate-charts b/scripts/validate-charts new file mode 100755 index 00000000000..655fcdca52a --- /dev/null +++ b/scripts/validate-charts @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -e + +info() { + echo '[INFO] ' "$@" +} + +error() { + echo '[ERROR] ' "$@" >&2 +} + +fatal() { + echo '[ERROR] ' "$@" >&2 + exit 1 +} + +cleanup() { + exit_code=$? + trap - EXIT INT + rm -rf /tmp/tmp.*.tar.gz + exit ${exit_code} +} +trap cleanup EXIT INT + + +download_chart() { + chart_version=$1 + chart_name=$2 + bootstrap=$3 + + chart_package=${chart_name%%-crd} + + chart_url=${CHART_REPO:="https://rke2-charts.rancher.io"}/assets/${chart_package}/${chart_name}-${chart_version:="v0.0.0"}.tgz + + chart_tmp=$(mktemp --suffix .tar.gz) + + curl -fsSL "${chart_url}" -o "${chart_tmp}" + + echo $chart_tmp +} + +check_system_registry() { + chart_version=$1 + chart_name=$2 + chart_tmp=$3 + + yaml_tmp=$(mktemp --suffix .yaml) + values="global.systemDefaultRegistry=my-registry,global.cattle.systemDefaultRegistry=my-registry,vCenter.clusterId=test-id" + helm template test-chart --set $values $chart_tmp > $yaml_tmp; + + awk '$1 ~ /^image:/ { + if( $2 !~ /my-registry/ && $2 !~ busybox) { + print $2 + } + } + ' $yaml_tmp +} + +check_airgap() { + chart_version=$1 + chart_name=$2 + chart_tmp=$3 + + # skip rancher-vsphere-cpi because it doesn't work well with `helm template` + if [[ $chart_name == 'rancher-vsphere-cpi' ]]; then + return 0 + fi + + yaml_tmp=$(mktemp --suffix .yaml) + values="vCenter.clusterId=test-id" + helm template test-chart --set $values $chart_tmp > $yaml_tmp; + + awk '$1 ~ /^image:/ { + gsub(/"/, "", $2) + gsub(/^docker.io/, "", $2) + print $2 + } + ' $yaml_tmp | \ + while read image + do + [ "$image" = "busybox" ] && continue + if ! grep -q $image scripts/build-images; then + echo $image + fi + done +} + +declare -A NO_SYSTEM_REGISTRY +declare -A NOT_FOUND + + +while read version filename bootstrap; do + chart_name=$(basename "${filename%%.yaml}") + chart_tmp=$(download_chart $version $chart_name $bootstrap) + + info "Validating chart $chart_name, version $version..." + + no_system_registry=$(check_system_registry $version $chart_name $chart_tmp) + if ! [ -z "$no_system_registry" ]; then + NO_SYSTEM_REGISTRY[$chart_name]=$no_system_registry + fi + + not_found=$(check_airgap $version $chart_name $chart_tmp) + if ! [ -z "$not_found" ]; then + NOT_FOUND[$chart_name]=$not_found + fi +done <<< $(yq e '.charts[] | [.version, .filename, .bootstrap] | join(" ")' charts/chart_versions.yaml) + +failed=0 + +if [ ${#NO_SYSTEM_REGISTRY[@]} -ge 1 ]; then + failed=1 + for chart in "${!NO_SYSTEM_REGISTRY[@]}" + do + error "Images not using global.systemDefaultRegistry in chart '$chart': ${NO_SYSTEM_REGISTRY[$chart]}" + done + error "Please use global.systemDefaultRegistry for above images" +fi + +if [ ${#NOT_FOUND[@]} -ge 1 ]; then + failed=1 + for chart in "${!NOT_FOUND[@]}" + do + error "Missing images for chart '$chart': ${NOT_FOUND[$chart]}" + done + error "Please include above images in build-images" +fi + +[ $failed = 1 ] && fatal "Please fix the issues above" + +exit 0