diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index b8c857c69..334f67d38 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -19,10 +19,17 @@ jobs: nix-shell ./scripts/k8s/shell.nix --run "echo" - name: BootStrap k8s cluster run: | - nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label" + nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label --dump-images" + - name: Install Helm Chart run: | nix-shell ./scripts/k8s/shell.nix --run "./scripts/helm/install.sh --dep-update --wait" + + - name: Container Image List + run: | + # Verifies if the list of images is accurate + nix-shell ./scripts/k8s/shell.nix --run "./scripts/helm/verify-kind-images.sh" + - name: The job has failed if: ${{ failure() }} run: | diff --git a/chart/dependencies-images.txt b/chart/dependencies-images.txt new file mode 100644 index 000000000..38a914f3a --- /dev/null +++ b/chart/dependencies-images.txt @@ -0,0 +1,3 @@ +# localpv-provisioner +# todo: we need a way to fetch this from the dependencies +docker.io/openebs/linux-utils:4.1.0 diff --git a/chart/images.txt b/chart/images.txt new file mode 100644 index 000000000..97482e236 --- /dev/null +++ b/chart/images.txt @@ -0,0 +1,37 @@ +docker.io/grafana/promtail:2.8.3 +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/mayastor-agent-ha-node:develop +docker.io/openebs/mayastor-csi-node:develop +registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/mayastor-metrics-exporter-io-engine:develop +docker.io/openebs/mayastor-io-engine:develop +openebs/provisioner-localpv:4.1.0 +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/mayastor-agent-core:develop +docker.io/openebs/mayastor-agent-ha-cluster:develop +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/mayastor-api-rest:develop +docker.io/openebs/alpine-sh:4.1.0 +registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 +registry.k8s.io/sig-storage/csi-attacher:v4.3.0 +registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3 +registry.k8s.io/sig-storage/snapshot-controller:v6.3.3 +registry.k8s.io/sig-storage/csi-resizer:v1.9.3 +docker.io/openebs/mayastor-csi-controller:develop +docker.io/openebs/mayastor-obs-callhome:develop +docker.io/openebs/mayastor-obs-callhome-stats:develop +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/mayastor-operator-diskpool:develop +docker.io/openebs/alpine-bash:4.1.0 +docker.io/bitnami/etcd:3.5.6-debian-11-r10 +docker.io/openebs/alpine-sh:4.1.0 +grafana/loki:2.6.1 +nats:2.9.17-alpine +natsio/nats-server-config-reloader:0.10.1 +natsio/prometheus-nats-exporter:0.11.0 +bats/bats:1.8.2 +natsio/nats-box:0.13.8 diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh index 66d6e5d6f..8ffa87faf 100755 --- a/scripts/helm/install.sh +++ b/scripts/helm/install.sh @@ -99,4 +99,4 @@ else set +x fi -kubectl get pods -n mayastor -o wide +kubectl get pods -n "$K8S_NAMESPACE" -o wide diff --git a/scripts/helm/template-images.sh b/scripts/helm/template-images.sh new file mode 100755 index 000000000..552449a1d --- /dev/null +++ b/scripts/helm/template-images.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." +CHART_DIR="$ROOT_DIR/chart" +IMAGES="$CHART_DIR/images.txt" + +EXIT_CODE= +DRY_RUN= +DEP_UPDATE= +HELM="helm" +ENABLE_ANALYTICS="eventing.enabled=true,obs.callhome.enabled=true,obs.callhome.sendReport=true,localpv-provisioner.analytics.enabled=true" + +help() { + cat <&2 +} + +die() { + local _return="${2:-1}" + echo_stderr "$1" + exit "${_return}" +} + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + shift;; + --exit-code) + EXIT_CODE="true" + shift;; + --dry-run) + DRY_RUN="true" + HELM="echo $HELM" + shift;; + --dependency-update) + DEP_UPDATE="true" + shift;; + *) + die "Unknown argument $1!" + shift;; + esac +done + +cd "$CHART_DIR" + +if [ "$DEP_UPDATE" = "true" ]; then + $HELM dependency update +fi + +$HELM template . --set "$ENABLE_ANALYTICS" | grep "image:" | awk '{ print $2 }' | tr -d \" > "$IMAGES" + +if [ "$EXIT_CODE" = "true" ]; then + git diff --exit-code "$IMAGES" +fi diff --git a/scripts/helm/verify-kind-images.sh b/scripts/helm/verify-kind-images.sh new file mode 100755 index 000000000..97777297c --- /dev/null +++ b/scripts/helm/verify-kind-images.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -euo pipefail + + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." +CHART_DIR="$ROOT_DIR/chart" +HELM_TPL_IMAGES="$CHART_DIR/images.txt" +HELM_DEP_IMAGES="$CHART_DIR/dependencies-images.txt" +TMP_KIND="/tmp/kind/mayastor" +KIND_IMAGES="kind-images.txt" +DOCKER="docker" + +echo_stderr() { + echo -e "${1}" >&2 +} + +die() { + local _return="${2:-1}" + echo_stderr "$1" + exit "${_return}" +} + +grep_image() { + local image="$1" + local file="$2" + + grep -q "^$image$" "$file" || grep -q "^${image##docker.io/}$" "$file" +} + +if ! [ -f "$HELM_TPL_IMAGES" ]; then + die "Missing helm template images, please generate them" +fi + +cd "$TMP_KIND" +for worker in kind-worker*; do + if ! [ -f "$worker/$KIND_IMAGES" ]; then + die "Missing $KIND_IMAGES file" + fi + + curr_images=$($DOCKER exec $worker crictl image | tail -n+2 | awk '{ print $1 ":" $2 }') + for image in ${curr_images[@]}; do + if grep_image "$image" "$worker/$KIND_IMAGES"; then + # if it's there before the install, then ignore it. + continue + fi + if ! grep_image "$image" "$HELM_TPL_IMAGES" && ! grep_image "$image" "$HELM_DEP_IMAGES"; then + echo "$image not found" + fi + done +done diff --git a/scripts/k8s/deployer.sh b/scripts/k8s/deployer.sh index c16c9c3d1..a67944c91 100755 --- a/scripts/k8s/deployer.sh +++ b/scripts/k8s/deployer.sh @@ -17,6 +17,7 @@ DOCKER="docker" HUGE_PAGES=1800 LABEL= SUDO=${SUDO:-"sudo"} +DUMP_KIND_IMG= help() { cat < Add 2MiB hugepages (Default: $HUGE_PAGES). --label Label worker nodes with the io-engine selector. + --dump-kind-images Dump the used images on the worker nodes to $TMP_KIND/$node-name/kind-images.txt Command: start Start the k8s cluster. @@ -95,6 +97,9 @@ while [ "$#" -gt 0 ]; do test $# -lt 1 && die "Missing hugepage number" HUGE_PAGES=$1 shift;; + --dump-kind-images) + DUMP_KIND_IMG="true" + shift;; --dry-run) if [ -z "$DRY_RUN" ]; then DRY_RUN="--dry-run" @@ -189,6 +194,10 @@ echo "HostIP: $host_ip" for node in ${nodes[@]}; do $DOCKER exec $node mount -o remount,rw /sys + if [ "$DUMP_KIND_IMG" = "true" ] && [ -z "$DRY_RUN" ]; then + $DOCKER exec $node crictl image | tail -n+2 | awk '{ print $1 ":" $2 }' > "$TMP_KIND/$node/kind-images.txt" + fi + # Note: this will go away if the node restarts... $DOCKER exec $node bash -c 'printf "'$host_ip' kvmhost\n" >> /etc/hosts' done