[7.17](backport #39295) Deploy/K8S pipeline post-migration fixes (#39487

) * Deploy/K8S pipeline post-migration fixes (#39295) * removed make.sh, updated check step * updated steps with env setup & removed setup from hook * updated setenv.sh * moved module to env * updated PR based on comments * updated common.sh * updated k8s env setup * removed tmp folder * added section for tests in BK (cherry picked from commit 8664cae) # Conflicts: # .buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml # .buildkite/scripts/setenv.sh * fixed conflicts * updated gh notifications --------- Co-authored-by: Olga Naydyonock <[email protected]>
elastic · May 10, 2024 · 01e4b02 · 01e4b02
1 parent e98b69c
commit 01e4b02
Show file tree

Hide file tree

Showing 10 changed files with 53 additions and 144 deletions.
diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
@@ -3,29 +3,40 @@
 env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  MODULE: "kubernetes"
+
+  # Other deps
+  ASDF_KIND_VERSION: "0.20.0"
 
 steps:
   - group: "Deploy/K8S"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
 
     steps:
     - label: "Checks"
-      command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+      command: |
+        set -euo pipefail
+        make -C deploy/kubernetes all
+        make check-no-changes
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-checks"
+            context: "deploy/k8s checks"
 
     - label: "K8S Test/K8S version: v1.21.14"
       key: "k8s-test-121"
       env:
         K8S_VERSION: "v1.21.14"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
       retry:
         automatic:
           - limit: 3
@@ -35,15 +46,19 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.21.14"
+            context: "deploy/k8s test v1.21.14"
 
     - label: "K8S Test/K8S version: v1.20.15"
       key: "k8s-test-120"
       env:
         K8S_VERSION: "v1.20.15"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
       retry:
         automatic:
           - limit: 3
@@ -53,15 +68,19 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.20.15"
+            context: "deploy/k8s test v1.20.15"
 
     - label: "K8S Test/K8S version: v1.19.16"
       key: "k8s-test-119"
       env:
         K8S_VERSION: "v1.19.16"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
       retry:
         automatic:
           - limit: 3
@@ -71,4 +90,4 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.19.16"
+            context: "deploy/k8s test v1.19.16"
diff --git a/.buildkite/deploy/kubernetes/scripts/install-kind.sh b/.buildkite/deploy/kubernetes/scripts/install-kind.sh
diff --git a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh
diff --git a/.buildkite/deploy/kubernetes/scripts/make.sh b/.buildkite/deploy/kubernetes/scripts/make.sh
diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
@@ -4,11 +4,15 @@ set -euo pipefail
 
 source .buildkite/env-scripts/util.sh
 
+export KUBECONFIG="${WORKSPACE}/kubecfg"
+export BIN="${WORKSPACE}/bin"
+add_bin_path
+
 echo "--- Installing kind & kubectl"
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh
+asdf plugin add kind
+asdf install kind $ASDF_KIND_VERSION
 
-echo "--- Setting up kind"
+echo "~~~ Setting up kind"
 max_retries=3
 timeout=5
 retries=0

diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
@@ -12,7 +12,6 @@ WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
-TMP_FOLDER="tmp.${REPO}"
 SNAPSHOT="true"
 PYTEST_ADDOPTS=""
 OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
@@ -38,7 +37,6 @@ export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
-export TMP_FOLDER
 export SNAPSHOT
 export PYTEST_ADDOPTS
 export OSS_MODULE_PATTERN

diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -88,9 +88,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
-  source .buildkite/env-scripts/env.sh
-  if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then
-    .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
-  fi
-fi
diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh
@@ -27,8 +27,8 @@ google_cloud_logout_active_account() {
 cleanup() {
   if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then
   echo "Deleting temporary files..."
-    if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then
-      rm -rf "${BIN}/${TMP_FOLDER}.*"
+    if [[ -n "${BIN:-}" ]]; then
+      rm -rf "${BIN}"
     fi
   echo "Done."
   fi

diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh
@@ -2,37 +2,20 @@
 
 set -euo pipefail
 
-REPO="beats"
-TMP_FOLDER="tmp.${REPO}"
-DOCKER_REGISTRY="docker.elastic.co"
-SETUP_GVM_VERSION="v0.5.1"
-DOCKER_COMPOSE_VERSION="1.21.0"
-DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
-SETUP_WIN_PYTHON_VERSION="3.11.0"
-NMAP_WIN_VERSION="7.12"           # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI)
+WORKSPACE=${WORKSPACE:-"$(pwd)"}
 GO_VERSION=$(cat .go-version)
-ASDF_MAGE_VERSION="1.15.0"
-ASDF_TERRAFORM_VERSION="1.0.2"
-PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
-PACKAGING_ARM_PLATFORMS="linux/arm64"
-AWS_REGION="eu-central-1"
-NODEJS_VERSION="18.17.1"
 
-export SETUP_GVM_VERSION
-export DOCKER_COMPOSE_VERSION
-export DOCKER_COMPOSE_VERSION_AARCH64
-export SETUP_WIN_PYTHON_VERSION
-export NMAP_WIN_VERSION
+export REPO="beats"
+export DOCKER_REGISTRY="docker.elastic.co"
+export SETUP_GVM_VERSION="v0.5.1"
+export DOCKER_COMPOSE_VERSION="1.21.0"
+export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
+
+export ASDF_NODEJS_VERSION="18.17.1"
+export AWS_REGION="eu-central-1"
+
+export WORKSPACE
 export GO_VERSION
-export ASDF_MAGE_VERSION
-export PACKAGING_PLATFORMS
-export PACKAGING_ARM_PLATFORMS
-export REPO
-export TMP_FOLDER
-export DOCKER_REGISTRY
-export ASDF_TERRAFORM_VERSION
-export AWS_REGION
-export NODEJS_VERSION
 
 exportVars() {
   local platform_type="$(uname)"