From b6f03256f75cb5c0ae1da71f2d704698f21c8d95 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 1 May 2024 16:43:24 +0300
Subject: [PATCH 01/26] updated heartbeat pipeline

---
 .buildkite/heartbeat/heartbeat-pipeline.yml | 80 +++++++++++++--------
 1 file changed, 51 insertions(+), 29 deletions(-)

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index dc27b45f1ac..8322e66dfdc 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -30,8 +30,9 @@ steps:
     key: "heartbeat-mandatory-tests"
 
     steps:
-      - label: ":ubuntu: Heartbeat Unit Tests"
+      - label: ":ubuntu: Unit Tests"
         command: |
+          set -euo pipefail
           cd heartbeat
           mage build unitTest
         retry:
@@ -46,10 +47,11 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Ubuntu Unit Tests"
+              context: "heartbeat: Ubuntu Unit Tests"
 
-      - label: ":rhel: Heartbeat Rhel9 Unit Tests"
+      - label: ":rhel: Rhel9 Unit Tests"
         command: |
+          set -euo pipefail
           cd heartbeat
           mage build unitTest
         retry:
@@ -64,9 +66,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Rhel9 Unit Tests"
+              context: "heartbeat: Rhel9 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-2016 Unit Tests"
+      - label: ":windows: Win-2016 Unit Tests"
         key: "windows-2016"
         command: |
           Set-Location -Path heartbeat
@@ -84,9 +86,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2016 Unit Tests"
+              context: "heartbeat: Win-2016 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-2022 Unit Test"
+      - label: ":windows: Win-2022 Unit Test"
         key: "windows-2022"
         command: |
           Set-Location -Path heartbeat
@@ -104,10 +106,15 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2022 Unit Tests"
+              context: "heartbeat: Win-2022 Unit Tests"
 
-      - label: ":ubuntu: Heartbeat Go Integration Tests"
+      - label: ":ubuntu: Go (Module) Integration Tests"
         command: |
+          set -euo pipefail
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet heartbeat
+          echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd heartbeat
           mage goIntegTest
         retry:
@@ -122,10 +129,15 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Go Integration Tests"
+              context: "heartbeat: Go Integration Tests"
 
-      - label: ":ubuntu: Heartbeat Python Integration Tests"
+      - label: ":ubuntu: Python (Module) Integration Tests"
         command: |
+          set -euo pipefail
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet heartbeat
+          echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd heartbeat
           mage pythonIntegTest
         retry:
@@ -140,16 +152,18 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Python Integration Tests"
+              context: "heartbeat: Python Integration Tests"
 
-  - group: "Heartbeat ARM Tests"
-    key: "heartbeat-extended-tests-arm"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+  - group: "Heartbeat Extended Tests"
+    key: "heartbeat-extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
 
     steps:
       - label: ":linux: Heartbeat ARM64 Unit Tests"
         key: "arm-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command: |
+          set -euo pipefail
           cd heartbeat
           mage build unitTest
         retry:
@@ -162,14 +176,16 @@ steps:
         artifact_paths: "heartbeat/build/*.xml"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Ubuntu ARM64 Unit Tests"
+              context: "heartbeat: Ubuntu ARM64 Unit Tests"
 
   - group: "Heartbeat Extended Testing MacOS"
     key: "heartbeat-extended-tests-macos"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+
     steps:
       - label: ":mac: Heartbeat MacOS Unit Tests"
         key: "macos-extended"
+        if:  build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -186,10 +202,11 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: MacOS Unit Tests"
+              context: "heartbeat: MacOS Unit Tests"
 
-      - label: ":mac: Heartbeat MacOS ARM Unit Tests"
+      - label: ":mac: MacOS ARM Unit Tests"
         key: "macos-arm-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -206,14 +223,14 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: MacOS ARM Unit Tests"
+              context: "heartbeat: MacOS ARM Unit Tests"
 
   - group: "Heartbeat Windows Extended Testing"
     key: "heartbeat-extended-tests-win"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
 
     steps:
-      - label: ":windows: Heartbeat Win-2019 Unit Tests"
+      - label: ":windows: Win-2019 Unit Tests"
         key: "heartbeat-win-extended-2019"
         command: |
           Set-Location -Path heartbeat
@@ -231,9 +248,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2019 Unit Tests"
+              context: "heartbeat: Win-2019 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-11 Unit Tests"
+      - label: ":windows: Win-11 Unit Tests"
         key: "heartbeat-windows-extended-11"
         command: |
           Set-Location -Path heartbeat
@@ -251,9 +268,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-11 Unit Tests"
+              context: "heartbeat: Win-11 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-10 Unit Tests"
+      - label: ":windows: Win-10 Unit Tests"
         key: "heartbeat-windows-extended-10"
         command: |
           Set-Location -Path heartbeat
@@ -271,21 +288,25 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-10 Unit Tests"
+              context: "heartbeat: Win-10 Unit Tests"
 
   - wait: ~
+    # with PRs, we want to run packaging only if mandatory tests succeed
+    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in mandatory tests
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     depends_on: "heartbeat-mandatory-tests"
 
   - group: "Heartbeat Packaging"
     key: "heartbeat-packaging"
     steps:
-      - label: ":ubuntu: Heartbeat Packaging Linux X86"
+      - label: ":ubuntu: Packaging Linux X86"
         key: "heartbeat-package-linux-x86"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
         command: |
+          set -euo pipefail
           cd heartbeat
           mage package
         retry:
@@ -298,15 +319,16 @@ steps:
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Packaging Ubuntu x86_64"
+              context: "heartbeat: Packaging Ubuntu x86_64"
 
-      - label: ":linux: Heartbeat Packaging Linux ARM"
+      - label: ":linux: Packaging Linux ARM"
         key: "heartbeat-package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
         command: |
+          set -euo pipefail
           cd heartbeat
           mage package
         retry:
@@ -319,4 +341,4 @@ steps:
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Packaging Ubuntu ARM"
+              context: "heartbeat: Packaging Ubuntu ARM"

From 33a6290d4c50a48ccc9d0970af99dd616f00caaa Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 8 May 2024 12:35:24 +0300
Subject: [PATCH 02/26] pr fixes

---
 .buildkite/heartbeat/heartbeat-pipeline.yml | 36 +++++++++------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 8322e66dfdc..876ac9c6b29 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -30,9 +30,8 @@ steps:
     key: "heartbeat-mandatory-tests"
 
     steps:
-      - label: ":ubuntu: Unit Tests"
+      - label: ":ubuntu: Heartbeat Unit Tests"
         command: |
-          set -euo pipefail
           cd heartbeat
           mage build unitTest
         retry:
@@ -49,9 +48,8 @@ steps:
           - github_commit_status:
               context: "heartbeat: Ubuntu Unit Tests"
 
-      - label: ":rhel: Rhel9 Unit Tests"
+      - label: ":rhel: Heartbeat Rhel9 Unit Tests"
         command: |
-          set -euo pipefail
           cd heartbeat
           mage build unitTest
         retry:
@@ -88,7 +86,7 @@ steps:
           - github_commit_status:
               context: "heartbeat: Win-2016 Unit Tests"
 
-      - label: ":windows: Win-2022 Unit Test"
+      - label: ":windows: Heartbeat Win-2022 Unit Test"
         key: "windows-2022"
         command: |
           Set-Location -Path heartbeat
@@ -108,7 +106,7 @@ steps:
           - github_commit_status:
               context: "heartbeat: Win-2022 Unit Tests"
 
-      - label: ":ubuntu: Go (Module) Integration Tests"
+      - label: ":ubuntu: Heartbeat Go (Module) Integration Tests"
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
@@ -131,7 +129,7 @@ steps:
           - github_commit_status:
               context: "heartbeat: Go Integration Tests"
 
-      - label: ":ubuntu: Python (Module) Integration Tests"
+      - label: ":ubuntu: Heartbeat Python (Module) Integration Tests"
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
@@ -159,11 +157,10 @@ steps:
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
 
     steps:
-      - label: ":linux: Heartbeat ARM64 Unit Tests"
+      - label: ":linux: Heartbeat arm64 Unit Tests"
         key: "arm-extended"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command: |
-          set -euo pipefail
           cd heartbeat
           mage build unitTest
         retry:
@@ -176,7 +173,7 @@ steps:
         artifact_paths: "heartbeat/build/*.xml"
         notify:
           - github_commit_status:
-              context: "heartbeat: Ubuntu ARM64 Unit Tests"
+              context: "heartbeat: Linux arm64 Unit Tests"
 
   - group: "Heartbeat Extended Testing MacOS"
     key: "heartbeat-extended-tests-macos"
@@ -184,6 +181,7 @@ steps:
 
     steps:
       - label: ":mac: Heartbeat MacOS Unit Tests"
+      - label: ":mac: macOS x86_64 Unit Tests"
         key: "macos-extended"
         if:  build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
@@ -202,9 +200,9 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "heartbeat: MacOS Unit Tests"
+              context: "heartbeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: MacOS ARM Unit Tests"
+      - label: ":mac: macOS arm64 Unit Tests"
         key: "macos-arm-extended"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
@@ -223,7 +221,7 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "heartbeat: MacOS ARM Unit Tests"
+              context: "heartbeat: macOS arm64 Unit Tests"
 
   - group: "Heartbeat Windows Extended Testing"
     key: "heartbeat-extended-tests-win"
@@ -300,13 +298,12 @@ steps:
   - group: "Heartbeat Packaging"
     key: "heartbeat-packaging"
     steps:
-      - label: ":ubuntu: Packaging Linux X86"
-        key: "heartbeat-package-linux-x86"
+      - label: ":ubuntu: Heartbeat Packaging Linux"
+        key: "heartbeat-package-linux"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
         command: |
-          set -euo pipefail
           cd heartbeat
           mage package
         retry:
@@ -319,16 +316,15 @@ steps:
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
-              context: "heartbeat: Packaging Ubuntu x86_64"
+              context: "heartbeat: Packaging Linux"
 
-      - label: ":linux: Packaging Linux ARM"
+      - label: ":linux: Heartbeat Packaging Linux arm64"
         key: "heartbeat-package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
         command: |
-          set -euo pipefail
           cd heartbeat
           mage package
         retry:
@@ -341,4 +337,4 @@ steps:
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         notify:
           - github_commit_status:
-              context: "heartbeat: Packaging Ubuntu ARM"
+              context: "heartbeat: Packaging Linux arm64"

From ca6ae9906c6a67f40998cfdcce8db1a08094d05f Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 14 May 2024 12:46:34 +0300
Subject: [PATCH 03/26] updated module test steps

---
 .buildkite/heartbeat/heartbeat-pipeline.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 876ac9c6b29..6799fba3e69 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -109,10 +109,13 @@ steps:
       - label: ":ubuntu: Heartbeat Go (Module) Integration Tests"
         command: |
           set -euo pipefail
+          echo "~~~ Installing @elastic/synthetics"
+          npm install -g @elastic/synthetics
+
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet heartbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+
           cd heartbeat
           mage goIntegTest
         retry:
@@ -135,7 +138,8 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet heartbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+
+          echo "~~~ Running tests"
           cd heartbeat
           mage pythonIntegTest
         retry:

From 21cee0077db1a7aeb97eda955fd765944795a34c Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 1 May 2024 16:43:24 +0300
Subject: [PATCH 04/26] updated heartbeat pipeline

---
 .buildkite/heartbeat/heartbeat-pipeline.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 6799fba3e69..d98d27b3f6d 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -106,7 +106,7 @@ steps:
           - github_commit_status:
               context: "heartbeat: Win-2022 Unit Tests"
 
-      - label: ":ubuntu: Heartbeat Go (Module) Integration Tests"
+      - label: ":ubuntu: Go (Module) Integration Tests"
         command: |
           set -euo pipefail
           echo "~~~ Installing @elastic/synthetics"
@@ -116,6 +116,7 @@ steps:
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet heartbeat
 
+          echo "~~~ Running tests"
           cd heartbeat
           mage goIntegTest
         retry:
@@ -132,7 +133,7 @@ steps:
           - github_commit_status:
               context: "heartbeat: Go Integration Tests"
 
-      - label: ":ubuntu: Heartbeat Python (Module) Integration Tests"
+      - label: ":ubuntu: Python (Module) Integration Tests"
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR

From 334498b1259c8f9844e006ae557112d13f3cc5d1 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 8 May 2024 12:35:24 +0300
Subject: [PATCH 05/26] pr fixes

---
 .buildkite/heartbeat/heartbeat-pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index d98d27b3f6d..7fe5cce11c1 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -106,7 +106,7 @@ steps:
           - github_commit_status:
               context: "heartbeat: Win-2022 Unit Tests"
 
-      - label: ":ubuntu: Go (Module) Integration Tests"
+      - label: ":ubuntu: Heartbeat Go (Module) Integration Tests"
         command: |
           set -euo pipefail
           echo "~~~ Installing @elastic/synthetics"
@@ -133,7 +133,7 @@ steps:
           - github_commit_status:
               context: "heartbeat: Go Integration Tests"
 
-      - label: ":ubuntu: Python (Module) Integration Tests"
+      - label: ":ubuntu: Heartbeat Python (Module) Integration Tests"
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR

From 21fbac4cbe5349a168c491067f89247da93f0d87 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 10 May 2024 12:11:38 +0300
Subject: [PATCH 06/26] updated env.sh

---
 .buildkite/env-scripts/env.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index 58624e300e6..dca5248f744 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -14,8 +14,6 @@ HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 SNAPSHOT="true"
 PYTEST_ADDOPTS=""
-OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
-XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 
 SETUP_GVM_VERSION="v0.5.1"
 ASDF_MAGE_VERSION="1.14.0"
@@ -32,15 +30,12 @@ GO_MOD_CHANGES=$(changeset_applies "^go.mod")
 KIND_VERSION="v0.20.0"
 KUBECONFIG="${WORKSPACE}/kubecfg"
 
-export REPO
 export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
 export SNAPSHOT
 export PYTEST_ADDOPTS
-export OSS_MODULE_PATTERN
-export XPACK_MODULE_PATTERN
 
 export SETUP_GVM_VERSION
 export ASDF_MAGE_VERSION

From a3c1cce3d3de11272c8df0c6275cb9e303bf1a0c Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 10 May 2024 15:12:51 +0300
Subject: [PATCH 07/26] updated scripts

---
 .buildkite/scripts/define_module.sh | 55 +++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 .buildkite/scripts/define_module.sh

diff --git a/.buildkite/scripts/define_module.sh b/.buildkite/scripts/define_module.sh
new file mode 100644
index 00000000000..4e175ed9a52
--- /dev/null
+++ b/.buildkite/scripts/define_module.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+BEAT_PATH=$1
+MODULE=''
+
+definePattern() {
+  pattern="${XPACK_MODULE_PATTERN}"
+
+  if [[ "$BEAT_PATH" == *"x-pack/"* ]]; then
+    pattern="${OSS_MODULE_PATTERN}"
+  fi
+}
+
+defineExclusions() {
+  local transformedDirectory=${BEAT_PATH//\//\\\/}
+  local exclusion="((?!^${transformedDirectory}\\/).)*\$"
+  exclude="^(${exclusion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)"
+}
+
+getGitMatchingGroup() {
+  local previousCommit
+  local matches
+  local match
+
+  local changeTarget=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-$BUILDKITE_BRANCH}
+  local baseCommit=$BUILDKITE_COMMIT
+  previousCommit=$(git rev-parse HEAD^)
+  from=$(echo ${changeTarget:+"origin/$changeTarget"}${previousCommit:-$baseCommit})
+  to=$baseCommit
+
+  matches=$(git diff --name-only "$from"..."$to" | grep -v "$exclude" | grep -oP "$pattern" | sort -u)
+
+  match=$(echo "$matches" | wc -w)
+
+  if [ "$match" -eq 1 ]; then
+    echo "$matches"
+  else
+    echo ''
+  fi
+}
+
+defineModule() {
+  cd "${BEAT_PATH}"
+  module=$(getGitMatchingGroup "$pattern" "$exclude")
+  if [ ! -f "$BEAT_PATH/module/${module}" ]; then
+    module=''
+  fi
+  cd - >/dev/null
+
+  echo "${module}"
+}

From d9f66f0bc507184d44ae74b1f8664a1d0ec6edfe Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 4 Dec 2023 15:36:12 +0200
Subject: [PATCH 08/26] wip: getting GH message

---
 .buildkite/filebeat/arm-tests.sh | 83 ++++++++++++++++++++++++++++++++
 .buildkite/filebeat/common.sh    | 14 ++++++
 2 files changed, 97 insertions(+)
 create mode 100644 .buildkite/filebeat/arm-tests.sh
 create mode 100644 .buildkite/filebeat/common.sh

diff --git a/.buildkite/filebeat/arm-tests.sh b/.buildkite/filebeat/arm-tests.sh
new file mode 100644
index 00000000000..660e91de668
--- /dev/null
+++ b/.buildkite/filebeat/arm-tests.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+WORKSPACE="$(pwd)"
+BIN="${WORKSPACE}/bin"
+
+add_bin_path() {
+    echo "Adding PATH to the environment variables..."
+    create_bin
+    export PATH="${PATH}:${BIN}"
+}
+
+with_go() {
+    local go_version="${1}"
+    echo "Setting up the Go environment..."
+    create_bin
+    check_platform_architecture
+    retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
+    export PATH="${PATH}:${BIN}"
+    chmod +x ${BIN}/gvm
+    eval "$(gvm "$go_version")"
+    go version
+    which go
+    export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin"
+}
+
+with_mage() {
+    local install_packages=(
+            "github.com/magefile/mage"
+            "github.com/elastic/go-licenser"
+            "golang.org/x/tools/cmd/goimports"
+            "github.com/jstemmer/go-junit-report"
+            "gotest.tools/gotestsum"
+    )
+    create_bin
+    for pkg in "${install_packages[@]}"; do
+        go install "${pkg}@latest"
+    done
+}
+
+create_bin() {
+    if [[ ! -d "${BIN}" ]]; then
+    mkdir -p ${BIN}
+    fi
+}
+
+check_platform_architecture() {
+# for downloading the GVM and Terraform packages
+  case "${HW_TYPE}" in
+   "x86_64")
+        arch_type="amd64"
+        ;;
+    "aarch64")
+        arch_type="arm64"
+        ;;
+    "arm64")
+        arch_type="arm64"
+        ;;
+    *)
+    echo "The current platform/OS type is unsupported yet"
+    ;;
+  esac
+}
+
+retry() {
+    local retries=$1
+    shift
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
diff --git a/.buildkite/filebeat/common.sh b/.buildkite/filebeat/common.sh
new file mode 100644
index 00000000000..0ac9279d8cd
--- /dev/null
+++ b/.buildkite/filebeat/common.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+are_files_changed() {
+  changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    return 0;
+  else
+    echo "WARN! No files changed in $changeset"
+    return 1;
+  fi
+}

From e839aa1ac7d2d644829aad6786b72f79fb821126 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 7 Dec 2023 11:35:07 +0200
Subject: [PATCH 09/26] added test scripts

---
 .buildkite/filebeat/common.sh                 | 14 --------------
 .buildkite/filebeat/scripts/arm-tests.sh      |  3 +++
 .../{arm-tests.sh => scripts/common.sh}       |  8 +++++++-
 .../filebeat/scripts/integration-tests.sh     | 19 +++++++++++++++++++
 4 files changed, 29 insertions(+), 15 deletions(-)
 delete mode 100644 .buildkite/filebeat/common.sh
 create mode 100644 .buildkite/filebeat/scripts/arm-tests.sh
 rename .buildkite/filebeat/{arm-tests.sh => scripts/common.sh} (90%)
 create mode 100755 .buildkite/filebeat/scripts/integration-tests.sh

diff --git a/.buildkite/filebeat/common.sh b/.buildkite/filebeat/common.sh
deleted file mode 100644
index 0ac9279d8cd..00000000000
--- a/.buildkite/filebeat/common.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-are_files_changed() {
-  changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    return 0;
-  else
-    echo "WARN! No files changed in $changeset"
-    return 1;
-  fi
-}
diff --git a/.buildkite/filebeat/scripts/arm-tests.sh b/.buildkite/filebeat/scripts/arm-tests.sh
new file mode 100644
index 00000000000..5b2371c6ed4
--- /dev/null
+++ b/.buildkite/filebeat/scripts/arm-tests.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
diff --git a/.buildkite/filebeat/arm-tests.sh b/.buildkite/filebeat/scripts/common.sh
similarity index 90%
rename from .buildkite/filebeat/arm-tests.sh
rename to .buildkite/filebeat/scripts/common.sh
index 660e91de668..4d2fd3aba6f 100644
--- a/.buildkite/filebeat/arm-tests.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -4,6 +4,12 @@ set -euo pipefail
 
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
+HW_TYPE="$(uname -m)"
+PLATFORM_TYPE="$(uname)"
+
+if [[ -z "${GOLANG_VERSION-""}" ]]; then
+    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
+fi
 
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
@@ -12,7 +18,7 @@ add_bin_path() {
 }
 
 with_go() {
-    local go_version="${1}"
+    local go_version="${GOLANG_VERSION}"
     echo "Setting up the Go environment..."
     create_bin
     check_platform_architecture
diff --git a/.buildkite/filebeat/scripts/integration-tests.sh b/.buildkite/filebeat/scripts/integration-tests.sh
new file mode 100755
index 00000000000..5060a18047f
--- /dev/null
+++ b/.buildkite/filebeat/scripts/integration-tests.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/filebeat/scripts/common.sh
+
+# ToDo - remove after Beats agent is created"
+echo ":: Setup Env ::"
+add_bin_path
+with_go
+with_mage
+# ToDo - end
+
+echo ":: Execute Integration Tests ::"
+sudo chmod -R go-w filebeat/
+
+cd filebeat
+umask 0022
+mage goIntegTest

From 25289813cfae1b97ec9b759b333794ee9bdb6638 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 13 Dec 2023 04:59:27 +0200
Subject: [PATCH 10/26] added windows tests

---
 .buildkite/filebeat/scripts/common.sh     |  8 ++++----
 .buildkite/filebeat/scripts/pre-common.sh | 10 ++++++++++
 filebeat/Jenkinsfile.yml                  |  2 +-
 3 files changed, 15 insertions(+), 5 deletions(-)
 create mode 100644 .buildkite/filebeat/scripts/pre-common.sh

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
index 4d2fd3aba6f..39c445b6acc 100644
--- a/.buildkite/filebeat/scripts/common.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -2,14 +2,14 @@
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
+#WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 
-if [[ -z "${GOLANG_VERSION-""}" ]]; then
-    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
-fi
+#if [[ -z "${GOLANG_VERSION-""}" ]]; then
+#    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
+#fi
 
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
diff --git a/.buildkite/filebeat/scripts/pre-common.sh b/.buildkite/filebeat/scripts/pre-common.sh
new file mode 100644
index 00000000000..12c69b771bc
--- /dev/null
+++ b/.buildkite/filebeat/scripts/pre-common.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+prepare_win() {
+  local os
+  os="$(uname)"
+  if [[ $os = MINGW* ]]; then
+    choco install mingw -y
+    choco install python --version=3.11.0 -y
+  fi
+}
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
index 127d82cecfd..4588823bd3c 100644
--- a/filebeat/Jenkinsfile.yml
+++ b/filebeat/Jenkinsfile.yml
@@ -42,7 +42,7 @@ stages:
           - "macos12 && x86_64"
         when:                  ## Override the top-level when.
             comments:
-                - "/test filebeat for macos"
+                  - "/test filebeat for macos"
             labels:
                 - "macOS"
             parameters:

From f3462371669bab30682f53fa29f9b37eb0f0d06b Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 3 Jan 2024 13:13:26 +0200
Subject: [PATCH 11/26] added packaging step

---
 .buildkite/filebeat/scripts/pre-common.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.buildkite/filebeat/scripts/pre-common.sh b/.buildkite/filebeat/scripts/pre-common.sh
index 12c69b771bc..f1a2bf5840e 100644
--- a/.buildkite/filebeat/scripts/pre-common.sh
+++ b/.buildkite/filebeat/scripts/pre-common.sh
@@ -8,3 +8,13 @@ prepare_win() {
     choco install python --version=3.11.0 -y
   fi
 }
+
+check_filebeat_changes() {
+  changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    export FILEBEAT_CHANGESET=true
+  else
+    export FILEBEAT_CHANGESET=false
+  fi
+}

From d9143369473651b3f486892c555283375f2e8ec3 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 4 Jan 2024 10:14:00 +0200
Subject: [PATCH 12/26] updated packaging execution conditions

---
 .buildkite/filebeat/scripts/common.sh     | 26 ++++++++++++++++++-----
 .buildkite/filebeat/scripts/pre-common.sh | 20 -----------------
 2 files changed, 21 insertions(+), 25 deletions(-)
 delete mode 100644 .buildkite/filebeat/scripts/pre-common.sh

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
index 39c445b6acc..943d3b98a6f 100644
--- a/.buildkite/filebeat/scripts/common.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -2,15 +2,11 @@
 
 set -euo pipefail
 
-#WORKSPACE="$(pwd)"
+WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 
-#if [[ -z "${GOLANG_VERSION-""}" ]]; then
-#    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
-#fi
-
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
     create_bin
@@ -87,3 +83,23 @@ retry() {
     done
     return 0
 }
+
+are_files_changed() {
+  changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    return 0;
+  else
+    echo "WARN! No files changed in $changeset"
+    return 1;
+  fi
+}
+
+prepare_win() {
+  local os
+  os="$(uname)"
+  if [[ $os = MINGW* ]]; then
+    choco install mingw -y
+    choco install python --version=3.11.0 -y
+  fi
+}
diff --git a/.buildkite/filebeat/scripts/pre-common.sh b/.buildkite/filebeat/scripts/pre-common.sh
deleted file mode 100644
index f1a2bf5840e..00000000000
--- a/.buildkite/filebeat/scripts/pre-common.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-prepare_win() {
-  local os
-  os="$(uname)"
-  if [[ $os = MINGW* ]]; then
-    choco install mingw -y
-    choco install python --version=3.11.0 -y
-  fi
-}
-
-check_filebeat_changes() {
-  changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    export FILEBEAT_CHANGESET=true
-  else
-    export FILEBEAT_CHANGESET=false
-  fi
-}

From 99704fd89cb9289f37161efdda53b94ecaeccd33 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 9 Jan 2024 22:24:32 +0200
Subject: [PATCH 13/26] win-test failure: updated artifact path

---
 .buildkite/filebeat/scripts/arm-tests.sh | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 .buildkite/filebeat/scripts/arm-tests.sh

diff --git a/.buildkite/filebeat/scripts/arm-tests.sh b/.buildkite/filebeat/scripts/arm-tests.sh
deleted file mode 100644
index 5b2371c6ed4..00000000000
--- a/.buildkite/filebeat/scripts/arm-tests.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail

From fe8a523e195db3cef2665e2c743652f83b5a4976 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 15 Jan 2024 11:58:12 +0200
Subject: [PATCH 14/26] pr fixes

---
 .../filebeat/scripts/integration-tests.sh     | 19 -------------------
 filebeat/Jenkinsfile.yml                      |  2 +-
 2 files changed, 1 insertion(+), 20 deletions(-)
 delete mode 100755 .buildkite/filebeat/scripts/integration-tests.sh

diff --git a/.buildkite/filebeat/scripts/integration-tests.sh b/.buildkite/filebeat/scripts/integration-tests.sh
deleted file mode 100755
index 5060a18047f..00000000000
--- a/.buildkite/filebeat/scripts/integration-tests.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/filebeat/scripts/common.sh
-
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
-
-echo ":: Execute Integration Tests ::"
-sudo chmod -R go-w filebeat/
-
-cd filebeat
-umask 0022
-mage goIntegTest
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
index 4588823bd3c..127d82cecfd 100644
--- a/filebeat/Jenkinsfile.yml
+++ b/filebeat/Jenkinsfile.yml
@@ -42,7 +42,7 @@ stages:
           - "macos12 && x86_64"
         when:                  ## Override the top-level when.
             comments:
-                  - "/test filebeat for macos"
+                - "/test filebeat for macos"
             labels:
                 - "macOS"
             parameters:

From d94a650a92737e118c6659b170654b748e99777f Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 16 Jan 2024 16:58:22 +0200
Subject: [PATCH 15/26] changed group_test.go

---
 .buildkite/hooks/pre-command | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 9a7228c5b2e..9a88715cded 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -52,6 +52,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then
     git config user.name "$github_username"
     git config user.email "$github_email"
   fi
+
+  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
+    ulimit -Sn 30000
+  fi
 fi
 
 ENABLED_BEATS_PIPELINES_SLUGS=(

From 811f4c97e53fdbf4fb0c5d00c7bc0c71fc0dcfec Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 17 Jan 2024 12:34:39 +0200
Subject: [PATCH 16/26] moved env setup to separate script

---
 .buildkite/filebeat/scripts/common.sh | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
index 943d3b98a6f..efbb81e66aa 100644
--- a/.buildkite/filebeat/scripts/common.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -2,10 +2,10 @@
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
-BIN="${WORKSPACE}/bin"
-HW_TYPE="$(uname -m)"
-PLATFORM_TYPE="$(uname)"
+#WORKSPACE="$(pwd)"
+#BIN="${WORKSPACE}/bin"
+#HW_TYPE="$(uname -m)"
+#PLATFORM_TYPE="$(uname)"
 
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
@@ -85,7 +85,7 @@ retry() {
 }
 
 are_files_changed() {
-  changeset=$1
+  local changeset=$1
 
   if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
     return 0;
@@ -95,11 +95,11 @@ are_files_changed() {
   fi
 }
 
-prepare_win() {
-  local os
-  os="$(uname)"
-  if [[ $os = MINGW* ]]; then
-    choco install mingw -y
-    choco install python --version=3.11.0 -y
-  fi
-}
+#prepare_win() {
+##  local os
+##  os="$(uname)"
+#  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
+#    choco install mingw -y
+#    choco install python --version=3.11.0 -y
+#  fi
+#}

From 2cf4de901d76a44da6cee5af53398bcb95907140 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 17 Jan 2024 15:48:22 +0200
Subject: [PATCH 17/26] added dynamic step for packaging

---
 .buildkite/filebeat/scripts/common.sh | 105 --------------------------
 1 file changed, 105 deletions(-)
 delete mode 100644 .buildkite/filebeat/scripts/common.sh

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
deleted file mode 100644
index efbb81e66aa..00000000000
--- a/.buildkite/filebeat/scripts/common.sh
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-#WORKSPACE="$(pwd)"
-#BIN="${WORKSPACE}/bin"
-#HW_TYPE="$(uname -m)"
-#PLATFORM_TYPE="$(uname)"
-
-add_bin_path() {
-    echo "Adding PATH to the environment variables..."
-    create_bin
-    export PATH="${PATH}:${BIN}"
-}
-
-with_go() {
-    local go_version="${GOLANG_VERSION}"
-    echo "Setting up the Go environment..."
-    create_bin
-    check_platform_architecture
-    retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
-    export PATH="${PATH}:${BIN}"
-    chmod +x ${BIN}/gvm
-    eval "$(gvm "$go_version")"
-    go version
-    which go
-    export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin"
-}
-
-with_mage() {
-    local install_packages=(
-            "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
-            "github.com/jstemmer/go-junit-report"
-            "gotest.tools/gotestsum"
-    )
-    create_bin
-    for pkg in "${install_packages[@]}"; do
-        go install "${pkg}@latest"
-    done
-}
-
-create_bin() {
-    if [[ ! -d "${BIN}" ]]; then
-    mkdir -p ${BIN}
-    fi
-}
-
-check_platform_architecture() {
-# for downloading the GVM and Terraform packages
-  case "${HW_TYPE}" in
-   "x86_64")
-        arch_type="amd64"
-        ;;
-    "aarch64")
-        arch_type="arm64"
-        ;;
-    "arm64")
-        arch_type="arm64"
-        ;;
-    *)
-    echo "The current platform/OS type is unsupported yet"
-    ;;
-  esac
-}
-
-retry() {
-    local retries=$1
-    shift
-    local count=0
-    until "$@"; do
-        exit=$?
-        wait=$((2 ** count))
-        count=$((count + 1))
-        if [ $count -lt "$retries" ]; then
-            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-            sleep $wait
-        else
-            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
-            return $exit
-        fi
-    done
-    return 0
-}
-
-are_files_changed() {
-  local changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    return 0;
-  else
-    echo "WARN! No files changed in $changeset"
-    return 1;
-  fi
-}
-
-#prepare_win() {
-##  local os
-##  os="$(uname)"
-#  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
-#    choco install mingw -y
-#    choco install python --version=3.11.0 -y
-#  fi
-#}

From 4f0ae8714ac4af2b5b30128e3634c3533843a74e Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 1 Feb 2024 11:33:38 +0200
Subject: [PATCH 18/26] added image tag and push

---
 .buildkite/env-scripts/macos-env.sh     |  0
 .buildkite/env-scripts/packaging-env.sh | 24 ++++++++++++++++
 .buildkite/env-scripts/util.sh          | 37 +++++++++++++++++++++++++
 3 files changed, 61 insertions(+)
 create mode 100644 .buildkite/env-scripts/macos-env.sh
 create mode 100644 .buildkite/env-scripts/packaging-env.sh

diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.buildkite/env-scripts/packaging-env.sh b/.buildkite/env-scripts/packaging-env.sh
new file mode 100644
index 00000000000..94ccb9296d3
--- /dev/null
+++ b/.buildkite/env-scripts/packaging-env.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+source .buildkite/env-scripts/util.sh
+
+docs_changeset="^.*\.(asciidoc|md)$
+deploy/kubernetes/.*-kubernetes.yaml"
+packaging_changeset="^dev-tools/packaging/
+^.go-version"
+
+# Env vars for Packaging stage
+DOCKER_REGISTRY="docker.elastic.co"
+SNAPSHOT=true
+VERSION=$(make get-version | tr -d '\n')
+ONLY_DOCS=$(changeset_applies "$docs_changeset")
+PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
+GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+
+export PACKAGING_CHANGES
+export ONLY_DOCS
+export GO_MOD_CHANGES
+export DOCKER_REGISTRY
+export SNAPSHOT
+export VERSION
+export REPO
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 68fd08a75df..261f2bbee2d 100755
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -102,3 +102,40 @@ set_git_config() {
   git config user.name "${GITHUB_USERNAME_SECRET}"
   git config user.email "${GITHUB_EMAIL_SECRET}"
 }
+
+changeset_applies() {
+  local changeset=$1
+  if are_files_changed "$changeset"; then
+    echo true
+  else
+    echo false
+  fi
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+      unset "$var"
+    fi
+  done
+}
+
+google_cloud_logout_active_account() {
+  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
+  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
+    echo "Logging out from GCP for active account"
+    gcloud auth revoke $active_account > /dev/null 2>&1
+  else
+    echo "No active GCP accounts found."
+  fi
+  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+    unset GOOGLE_APPLICATION_CREDENTIALS
+    cleanup
+  fi
+}
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${BIN}/${TMP_FOLDER}.*
+  echo "Done."
+}

From c48f933c89849379e9fc8cc498c101411df97fc6 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 6 Feb 2024 12:21:19 +0200
Subject: [PATCH 19/26] changed ubuntu agent

---
 .buildkite/env-scripts/macos-env.sh           |  8 +++++
 .../filebeat/scripts/packaging/package.sh     | 29 +++++++++++++++++++
 .../scripts/packaging}/packaging-env.sh       | 11 ++++++-
 3 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100755 .buildkite/filebeat/scripts/packaging/package.sh
 rename .buildkite/{env-scripts => filebeat/scripts/packaging}/packaging-env.sh (70%)
 mode change 100644 => 100755

diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh
index e69de29bb2d..ac1486b64fd 100644
--- a/.buildkite/env-scripts/macos-env.sh
+++ b/.buildkite/env-scripts/macos-env.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo ":: Setting larger ulimit on MacOS ::"
+  # To bypass file descriptor errors like "Too many open files error" on MacOS
+  ulimit -Sn 50000
+  echo ":: ULIMIT :: $(ulimit -n)"
+fi
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
new file mode 100755
index 00000000000..9cce3b1dc66
--- /dev/null
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo ":: Start Packaging ::"
+mage -d filebeat package
+
+calculate_tags() {
+  if [ "$snapshot" = true ]; then
+    aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
+    sourceTag+=${IMG_POSTFIX}
+  fi
+
+  local tags="${BUILDKITE_COMMIT}"
+
+#  if isPR; then
+#    tags+=("pr-${CHANGE_ID}")
+#  else
+#    tags+=("${sourceTag}")
+#  fi
+#
+#  if ! isPR && [ -n "$(aliasVersion)" ]; then
+#    tags+=("${aliasVersion}")
+#  fi
+#
+#  echo "${tags[@]}"
+}
+
+#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
diff --git a/.buildkite/env-scripts/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
old mode 100644
new mode 100755
similarity index 70%
rename from .buildkite/env-scripts/packaging-env.sh
rename to .buildkite/filebeat/scripts/packaging/packaging-env.sh
index 94ccb9296d3..ad4957bd911
--- a/.buildkite/env-scripts/packaging-env.sh
+++ b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
@@ -7,13 +7,14 @@ deploy/kubernetes/.*-kubernetes.yaml"
 packaging_changeset="^dev-tools/packaging/
 ^.go-version"
 
-# Env vars for Packaging stage
 DOCKER_REGISTRY="docker.elastic.co"
 SNAPSHOT=true
 VERSION=$(make get-version | tr -d '\n')
 ONLY_DOCS=$(changeset_applies "$docs_changeset")
 PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
 GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+# Change the postfix to -SNAPSHOT, once Jenkins is disabled
+IMG_POSTFIX="-BK-SNAPSHOT"
 
 export PACKAGING_CHANGES
 export ONLY_DOCS
@@ -22,3 +23,11 @@ export DOCKER_REGISTRY
 export SNAPSHOT
 export VERSION
 export REPO
+export IMG_POSTFIX
+
+set_git_config() {
+  git config user.name "${GITHUB_USERNAME_SECRET}"
+  git config user.email "${GITHUB_EMAIL_SECRET}"
+}
+
+set_git_config

From 35bb01fbe2484aa0607b50f7cd9e9d12cc239e16 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 6 Feb 2024 20:21:45 +0200
Subject: [PATCH 20/26] added secrets

---
 .buildkite/filebeat/scripts/packaging/package.sh       | 7 +++++++
 .buildkite/filebeat/scripts/packaging/packaging-env.sh | 7 -------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 9cce3b1dc66..29faa449519 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -27,3 +27,10 @@ calculate_tags() {
 }
 
 #buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
+
+#set_git_config() {
+#  git config user.name "${GITHUB_USERNAME_SECRET}"
+#  git config user.email "${GITHUB_EMAIL_SECRET}"
+#}
+#
+#set_git_config
diff --git a/.buildkite/filebeat/scripts/packaging/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
index ad4957bd911..ab8f8f68257 100755
--- a/.buildkite/filebeat/scripts/packaging/packaging-env.sh
+++ b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
@@ -24,10 +24,3 @@ export SNAPSHOT
 export VERSION
 export REPO
 export IMG_POSTFIX
-
-set_git_config() {
-  git config user.name "${GITHUB_USERNAME_SECRET}"
-  git config user.email "${GITHUB_EMAIL_SECRET}"
-}
-
-set_git_config

From 0facfabf067918799169578122ecf7bab37997ae Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 6 Feb 2024 21:07:57 +0200
Subject: [PATCH 21/26] added tag definition

---
 .buildkite/env-scripts/macos-env.sh           |  8 --
 .../filebeat/scripts/packaging/package.sh     | 73 +++++++++++++------
 2 files changed, 50 insertions(+), 31 deletions(-)
 delete mode 100644 .buildkite/env-scripts/macos-env.sh

diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh
deleted file mode 100644
index ac1486b64fd..00000000000
--- a/.buildkite/env-scripts/macos-env.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ $PLATFORM_TYPE == Darwin* ]]; then
-  echo ":: Setting larger ulimit on MacOS ::"
-  # To bypass file descriptor errors like "Too many open files error" on MacOS
-  ulimit -Sn 50000
-  echo ":: ULIMIT :: $(ulimit -n)"
-fi
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 29faa449519..6546db82df3 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -2,35 +2,62 @@
 
 set -euo pipefail
 
-echo ":: Start Packaging ::"
-mage -d filebeat package
+#source .buildkite/env-scripts/linux-env.sh
+source .buildkite/env-scripts/util.sh
 
-calculate_tags() {
-  if [ "$snapshot" = true ]; then
+#echo "--- Creating package"
+#mage -d filebeat package
+
+
+TAGS=("${BUILDKITE_COMMIT}")
+
+VARIANTS=("" "-ubi" "-oss")
+
+# IMAGES
+SOURCE_NAMESPACE="beats"
+BEAT_NAME="filebeat"
+SOURCE="beats/${BEAT_NAME}${variant}"
+TARGET="observability-ci/${BEAT_NAME}"
+
+# ARGS
+REGISTRY="docker.elastic.co"
+SNAPSHOT=true
+VERSION="$(make get-version)",
+images: images
+
+#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
+
+#IMAGES -> foreach = tag and push
+# registry: ${REGISTRY},
+# sourceTag: calculate_tags->sourceTag,
+# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
+# source: ${SOURCE},
+# target: ${TARGET}
+
+
+define_tags() {
+echo "--- Defined tags"
+  if [ "$SNAPSHOT" = true ]; then
     aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
-    sourceTag+=${IMG_POSTFIX}
+    sourceTag+="${VERSION}${IMG_POSTFIX}"
   fi
 
-  local tags="${BUILDKITE_COMMIT}"
-
-#  if isPR; then
-#    tags+=("pr-${CHANGE_ID}")
-#  else
-#    tags+=("${sourceTag}")
-#  fi
-#
-#  if ! isPR && [ -n "$(aliasVersion)" ]; then
-#    tags+=("${aliasVersion}")
-#  fi
-#
-#  echo "${tags[@]}"
+  if is_pr; then
+    TAGS+=("pr-${GITHUB_PR_NUMBER}")
+  else
+    TAGS+=("${sourceTag}" "${aliasVersion}")
+  fi
+
+  local tag=""
+  for tag in "${TAGS[@]}"; do
+    echo "$tag"
+  done
 }
 
 #buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
 
-#set_git_config() {
-#  git config user.name "${GITHUB_USERNAME_SECRET}"
-#  git config user.email "${GITHUB_EMAIL_SECRET}"
-#}
-#
+echo "--- Calculating tags"
+daefine_tags
+
+#echo "--- Setting git config"
 #set_git_config

From 1a46e70065207bf249b85d8060ae282aba2168e6 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 9 Feb 2024 12:06:18 +0200
Subject: [PATCH 22/26] added docker img params

---
 .../filebeat/scripts/packaging/package.sh     | 79 +++++++++----------
 .../scripts/packaging/packaging-env.sh        | 17 ++--
 2 files changed, 51 insertions(+), 45 deletions(-)

diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 6546db82df3..55a3159d8cb 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -5,59 +5,58 @@ set -euo pipefail
 #source .buildkite/env-scripts/linux-env.sh
 source .buildkite/env-scripts/util.sh
 
-#echo "--- Creating package"
-#mage -d filebeat package
-
-
-TAGS=("${BUILDKITE_COMMIT}")
-
+IMG_POSTFIX="-BK-SNAPSHOT"
 VARIANTS=("" "-ubi" "-oss")
-
-# IMAGES
-SOURCE_NAMESPACE="beats"
+VERSION="$(make get-version)"
+SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
 BEAT_NAME="filebeat"
-SOURCE="beats/${BEAT_NAME}${variant}"
 TARGET="observability-ci/${BEAT_NAME}"
 
-# ARGS
-REGISTRY="docker.elastic.co"
-SNAPSHOT=true
-VERSION="$(make get-version)",
-images: images
-
-#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
-
-#IMAGES -> foreach = tag and push
-# registry: ${REGISTRY},
-# sourceTag: calculate_tags->sourceTag,
-# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
-# source: ${SOURCE},
-# target: ${TARGET}
-
-
 define_tags() {
-echo "--- Defined tags"
-  if [ "$SNAPSHOT" = true ]; then
-    aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
-    sourceTag+="${VERSION}${IMG_POSTFIX}"
-  fi
+  aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
+  tags=("${BUILDKITE_COMMIT}")
 
   if is_pr; then
-    TAGS+=("pr-${GITHUB_PR_NUMBER}")
+    tags+=("pr-${GITHUB_PR_NUMBER}")
   else
-    TAGS+=("${sourceTag}" "${aliasVersion}")
+    tags+=("${SOURCE_TAG}" "${aliasVersion}")
   fi
+}
 
-  local tag=""
-  for tag in "${TAGS[@]}"; do
-    echo "$tag"
-  done
+check_is_arm() {
+  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
+    is_arm="-arm"
+  else
+    is_arm=""
+  fi
 }
 
-#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
+define_tags
+
+for variant in "${VARIANTS[@]}"; do
+  echo "--- PARAMS for variant: $variant"
+
+  check_is_arm
+  registry=${DOCKER_REGISTRY}
+  sourceTag=$SOURCE_TAG
+  source="beats/${BEAT_NAME}${variant}"
+  target=$TARGET
+
+  echo "Registry: $registry"
+  echo "Source: $source"
+  echo "Source tag: $sourceTag"
+  echo "Target: $target"
 
-echo "--- Calculating tags"
-daefine_tags
+  for tag in "${tags[@]}"; do
+    targetTag=$tag${is_arm}
+    echo "Target tag: $targetTag"
+  done
+done
+
+#echo "--- Creating package"
+#mage -d filebeat package
 
 #echo "--- Setting git config"
 #set_git_config
+
+#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
diff --git a/.buildkite/filebeat/scripts/packaging/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
index ab8f8f68257..02c942361b4 100755
--- a/.buildkite/filebeat/scripts/packaging/packaging-env.sh
+++ b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
@@ -7,20 +7,27 @@ deploy/kubernetes/.*-kubernetes.yaml"
 packaging_changeset="^dev-tools/packaging/
 ^.go-version"
 
-DOCKER_REGISTRY="docker.elastic.co"
-SNAPSHOT=true
 VERSION=$(make get-version | tr -d '\n')
 ONLY_DOCS=$(changeset_applies "$docs_changeset")
 PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
 GO_MOD_CHANGES=$(changeset_applies "^go.mod")
 # Change the postfix to -SNAPSHOT, once Jenkins is disabled
-IMG_POSTFIX="-BK-SNAPSHOT"
+
 
 export PACKAGING_CHANGES
 export ONLY_DOCS
 export GO_MOD_CHANGES
 export DOCKER_REGISTRY
-export SNAPSHOT
 export VERSION
 export REPO
-export IMG_POSTFIX
+#export IMG_POSTFIX
+
+
+#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
+
+#IMAGES -> foreach = tag and push
+# registry: ${REGISTRY},
+# sourceTag: calculate_tags->sourceTag,
+# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
+# source: ${SOURCE},
+# target: ${TARGET}

From 9252aab3a84f385c76f4ddac362070d1b891304d Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 12 Feb 2024 13:42:42 +0200
Subject: [PATCH 23/26] added tag and push

---
 .../scripts/packaging/package-util.sh         | 30 ++++++++
 .../filebeat/scripts/packaging/package.sh     | 71 +++++++++----------
 2 files changed, 62 insertions(+), 39 deletions(-)
 create mode 100755 .buildkite/filebeat/scripts/packaging/package-util.sh

diff --git a/.buildkite/filebeat/scripts/packaging/package-util.sh b/.buildkite/filebeat/scripts/packaging/package-util.sh
new file mode 100755
index 00000000000..cb409e8d484
--- /dev/null
+++ b/.buildkite/filebeat/scripts/packaging/package-util.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+is_pr() {
+  if [[ $BUILDKITE_PULL_REQUEST != false ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+define_tags() {
+  aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
+  tags=("${BUILDKITE_COMMIT}")
+
+  if is_pr; then
+    tags+=("pr-${GITHUB_PR_NUMBER}")
+  else
+    tags+=("${SOURCE_TAG}" "${aliasVersion}")
+  fi
+}
+
+check_is_arm() {
+  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
+    is_arm="-arm"
+  else
+    is_arm=""
+  fi
+}
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 55a3159d8cb..d7ec456f615 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -2,61 +2,54 @@
 
 set -euo pipefail
 
-#source .buildkite/env-scripts/linux-env.sh
-source .buildkite/env-scripts/util.sh
+source .buildkite/env-scripts/linux-env.sh
+source .buildkite/filebeat/scripts/packaging/package-util.sh
 
-IMG_POSTFIX="-BK-SNAPSHOT"
+IMG_POSTFIX="-SNAPSHOT"
 VARIANTS=("" "-ubi" "-oss")
 VERSION="$(make get-version)"
 SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
 BEAT_NAME="filebeat"
 TARGET="observability-ci/${BEAT_NAME}"
+# Remove following once beats fully migrated
+BK_IMG_POSTFIX="-BK-SNAPSHOT"
+BK_SOURCE_TAG+="${VERSION}${BK_IMG_POSTFIX}"
 
-define_tags() {
-  aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
-  tags=("${BUILDKITE_COMMIT}")
+echo "--- Creating package"
+mage -d filebeat package
 
-  if is_pr; then
-    tags+=("pr-${GITHUB_PR_NUMBER}")
-  else
-    tags+=("${SOURCE_TAG}" "${aliasVersion}")
-  fi
-}
+echo "--- Distribution list"
+ls -la filebeat/build/distributions
 
-check_is_arm() {
-  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
-    is_arm="-arm"
-  else
-    is_arm=""
-  fi
-}
+echo "--- Docker image list"
+docker images
 
 define_tags
+check_is_arm
 
 for variant in "${VARIANTS[@]}"; do
-  echo "--- PARAMS for variant: $variant"
 
-  check_is_arm
-  registry=${DOCKER_REGISTRY}
-  sourceTag=$SOURCE_TAG
   source="beats/${BEAT_NAME}${variant}"
-  target=$TARGET
-
-  echo "Registry: $registry"
-  echo "Source: $source"
-  echo "Source tag: $sourceTag"
-  echo "Target: $target"
 
   for tag in "${tags[@]}"; do
-    targetTag=$tag${is_arm}
-    echo "Target tag: $targetTag"
+  targetTag=$tag${is_arm}
+
+  sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
+  targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+
+  if docker image inspect "${sourceName}" &>/dev/null; then
+    echo "--- Tag & Push"
+    echo "Source name: $sourceName"
+    echo "Target name: $targetName"
+
+    # Remove following lines once beats fully migrated
+    bkSourceName="${DOCKER_REGISTRY}/${source}:${BK_SOURCE_TAG}"
+    docker tag "$sourceName" "$bkSourceName"
+    # Replace bkSourceName to sourceName once beats fully migrated
+    docker tag "${bkSourceName}" "${targetName}"
+#    docker push "${targetName}"
+  else
+    echo "Docker image ${sourceName} does not exist"
+  fi
   done
 done
-
-#echo "--- Creating package"
-#mage -d filebeat package
-
-#echo "--- Setting git config"
-#set_git_config
-
-#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'

From 2d43046d4219bf68f1e325d7403872fe16e42d9f Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 14 Feb 2024 12:28:16 +0200
Subject: [PATCH 24/26] added distribution upload

---
 .../filebeat/scripts/packaging/package.sh     | 26 +++++++------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index d7ec456f615..44e91044576 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -11,15 +11,13 @@ VERSION="$(make get-version)"
 SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
 BEAT_NAME="filebeat"
 TARGET="observability-ci/${BEAT_NAME}"
-# Remove following once beats fully migrated
-BK_IMG_POSTFIX="-BK-SNAPSHOT"
-BK_SOURCE_TAG+="${VERSION}${BK_IMG_POSTFIX}"
 
 echo "--- Creating package"
 mage -d filebeat package
 
 echo "--- Distribution list"
-ls -la filebeat/build/distributions
+dir="filebeat/build/distributions"
+buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
 
 echo "--- Docker image list"
 docker images
@@ -27,8 +25,8 @@ docker images
 define_tags
 check_is_arm
 
+echo "--- Tag & Push"
 for variant in "${VARIANTS[@]}"; do
-
   source="beats/${BEAT_NAME}${variant}"
 
   for tag in "${tags[@]}"; do
@@ -36,20 +34,16 @@ for variant in "${VARIANTS[@]}"; do
 
   sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
   targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+  # Remove following lines once beats fully migrated
+  targetName="${targetName}-buildkite"
 
   if docker image inspect "${sourceName}" &>/dev/null; then
-    echo "--- Tag & Push"
-    echo "Source name: $sourceName"
-    echo "Target name: $targetName"
-
-    # Remove following lines once beats fully migrated
-    bkSourceName="${DOCKER_REGISTRY}/${source}:${BK_SOURCE_TAG}"
-    docker tag "$sourceName" "$bkSourceName"
-    # Replace bkSourceName to sourceName once beats fully migrated
-    docker tag "${bkSourceName}" "${targetName}"
-#    docker push "${targetName}"
+    echo "Source name: $sourceName Target name: $targetName"
+    docker tag "$sourceName" "$targetName"
+#    docker push "$targetName"
+
   else
     echo "Docker image ${sourceName} does not exist"
   fi
-  done
+done
 done

From 9c8e6b7bdbe4f8423dbe9e8369fe6678f37f461e Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 14 Feb 2024 15:34:06 +0200
Subject: [PATCH 25/26] do-not-merge: testing ubuntu agent

---
 .../filebeat/scripts/packaging/package.sh     |  64 +++++++++---------
 .buildkite/scripts/define_module.sh           |   6 +-
 filebeat/filebeat_windows_amd64.syso          | Bin 1072 -> 0 bytes
 3 files changed, 35 insertions(+), 35 deletions(-)
 mode change 100644 => 100755 .buildkite/scripts/define_module.sh
 delete mode 100644 filebeat/filebeat_windows_amd64.syso

diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 44e91044576..f338e97c0b4 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -15,35 +15,35 @@ TARGET="observability-ci/${BEAT_NAME}"
 echo "--- Creating package"
 mage -d filebeat package
 
-echo "--- Distribution list"
-dir="filebeat/build/distributions"
-buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
-
-echo "--- Docker image list"
-docker images
-
-define_tags
-check_is_arm
-
-echo "--- Tag & Push"
-for variant in "${VARIANTS[@]}"; do
-  source="beats/${BEAT_NAME}${variant}"
-
-  for tag in "${tags[@]}"; do
-  targetTag=$tag${is_arm}
-
-  sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
-  targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
-  # Remove following lines once beats fully migrated
-  targetName="${targetName}-buildkite"
-
-  if docker image inspect "${sourceName}" &>/dev/null; then
-    echo "Source name: $sourceName Target name: $targetName"
-    docker tag "$sourceName" "$targetName"
-#    docker push "$targetName"
-
-  else
-    echo "Docker image ${sourceName} does not exist"
-  fi
-done
-done
+#echo "--- Distribution list"
+#dir="filebeat/build/distributions"
+#buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
+#
+#echo "--- Docker image list"
+#docker images
+#
+#define_tags
+#check_is_arm
+#
+#echo "--- Tag & Push"
+#for variant in "${VARIANTS[@]}"; do
+#  source="beats/${BEAT_NAME}${variant}"
+#
+#  for tag in "${tags[@]}"; do
+#  targetTag=$tag${is_arm}
+#
+#  sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
+#  targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+#  # Remove following lines once beats fully migrated
+#  targetName="${targetName}-buildkite"
+#
+#  if docker image inspect "${sourceName}" &>/dev/null; then
+#    echo "Source name: $sourceName Target name: $targetName"
+#    docker tag "$sourceName" "$targetName"
+##    docker push "$targetName"
+#
+#  else
+#    echo "Docker image ${sourceName} does not exist"
+#  fi
+#done
+#done
diff --git a/.buildkite/scripts/define_module.sh b/.buildkite/scripts/define_module.sh
old mode 100644
new mode 100755
index 4e175ed9a52..d7187b58901
--- a/.buildkite/scripts/define_module.sh
+++ b/.buildkite/scripts/define_module.sh
@@ -45,11 +45,11 @@ getGitMatchingGroup() {
 
 defineModule() {
   cd "${BEAT_PATH}"
-  module=$(getGitMatchingGroup "$pattern" "$exclude")
+  NEW_MODULE=$(getGitMatchingGroup "$pattern" "$exclude")
   if [ ! -f "$BEAT_PATH/module/${module}" ]; then
-    module=''
+    NEW_MODULE=''
   fi
   cd - >/dev/null
 
-  echo "${module}"
+  export NEW_MODULE
 }
diff --git a/filebeat/filebeat_windows_amd64.syso b/filebeat/filebeat_windows_amd64.syso
deleted file mode 100644
index c52af94f8e059275dff851e701e42fafefdf4132..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1072
zcmZvcPiqrV6vfY&fOb*nQd|w_QbN*yQBY7!i-l0dN^vLCB#i?x6Ot*k(1j~MPnT}|
z7B2f8`VGY2dDEAF_$Kq-efOPr?!D)|N&jaew<mR}vg%1=I-B-_x9a}Xq*s_*VBCy*
z6Mmk>mS_v}vGgN)sxb-p=70Pf@Vy80h4-2mv0c}8F;8`(uk;7{gmt73I%a-Ee~9h&
zyMFiA*=*`jn8z_p?z^g{h3EExcburyRJq0)!j@~K0nB5as{r~dV47Neq)R-B-huhf
zGXP&|2EKRi$(g^_w)WLlrWV!?<DOpX42+*P)&OrC%sKrIj4k|UU_5K7kLCFuh<!X(
zqDg%*?gYlMu23NhIk>U|^3TYmq!ORo4LK(Bx;jLh;di~0fGJ>bi<LX4jFl9etLihR
za2BYOl4qrleI>qB-LT?#j`|Hx0ei~6oN+*`lHFHx{AzxVk%8;6vk~%W>futpr>sxm
z&0($bd;zY777dhe8|f+dF1R5~qi`Y<;<>^htR)zq(yA|A)9HDr#EQ9QXO-$LH>1nA
zJ&)Dlb|u`mirY8F5u<XR`6g(tQq;16_cE-AGu(jVBOC=9T?BoW&o8pYy{>ULXtKIz
vU72-VI_yjMVytfDT-<8unfw8NX4Q9SMjbx&h0c!a@6oNM<$r-Bc)a@`Kf{eH


From 2cddce9121f6982edeba35738f5bb8a52cc8d197 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 10 May 2024 17:11:09 +0300
Subject: [PATCH 26/26] test module definition

---
 .buildkite/env-scripts/env.sh                 |  54 --
 .buildkite/env-scripts/util.sh                |  37 --
 .../scripts/packaging/package-util.sh         |  30 -
 .../filebeat/scripts/packaging/package.sh     |  49 --
 .../scripts/packaging/packaging-env.sh        |  33 -
 .buildkite/heartbeat/heartbeat-pipeline.yml   | 570 +++++++++---------
 .buildkite/hooks/pre-command                  |  49 --
 .buildkite/metricbeat/pipeline.yml            | 546 ++++++++---------
 .buildkite/scripts/changesets.sh              |   4 +-
 .buildkite/scripts/define_module.sh           |  65 +-
 .buildkite/scripts/moduleDefinition/go.mod    |   3 +
 .buildkite/scripts/moduleDefinition/main.go   | 102 ++++
 metricbeat/module/linux/deleteme.asciidoc     |   0
 metricbeat/module/linux/deleteme.go           |   0
 metricbeat/module/linux/deleteme.png          |   0
 .../module/linux/deleteme1/deletemeagain.go   |   0
 .../logstash/logstash_integration_test.go     |   1 +
 .../module/deleteme/subdir/deletemeagain.go   |   1 +
 x-pack/auditbeat/module/system/deleteme.go    |   1 +
 x-pack/metricbeat/module/deleteme.go          |   1 +
 x-pack/metricbeat/module/deleteme/deletme.go  |   1 +
 .../module/deleteme/subdir/deletemeagain.go   |   1 +
 22 files changed, 713 insertions(+), 835 deletions(-)
 delete mode 100644 .buildkite/env-scripts/env.sh
 delete mode 100755 .buildkite/filebeat/scripts/packaging/package-util.sh
 delete mode 100755 .buildkite/filebeat/scripts/packaging/package.sh
 delete mode 100755 .buildkite/filebeat/scripts/packaging/packaging-env.sh
 create mode 100644 .buildkite/scripts/moduleDefinition/go.mod
 create mode 100755 .buildkite/scripts/moduleDefinition/main.go
 create mode 100644 metricbeat/module/linux/deleteme.asciidoc
 create mode 100644 metricbeat/module/linux/deleteme.go
 create mode 100644 metricbeat/module/linux/deleteme.png
 create mode 100644 metricbeat/module/linux/deleteme1/deletemeagain.go
 create mode 100644 x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go
 create mode 100644 x-pack/auditbeat/module/system/deleteme.go
 create mode 100644 x-pack/metricbeat/module/deleteme.go
 create mode 100644 x-pack/metricbeat/module/deleteme/deletme.go
 create mode 100644 x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go

diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
deleted file mode 100644
index dca5248f744..00000000000
--- a/.buildkite/env-scripts/env.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env bash
-
-source .buildkite/env-scripts/util.sh
-
-DOCS_CHANGESET="^.*\.(asciidoc|md)$
-deploy/kubernetes/.*-kubernetes.yaml"
-PACKAGING_CHANGESET="^dev-tools/packaging/
-^.go-version"
-
-REPO="beats"
-WORKSPACE="$(pwd)"
-BIN="${WORKSPACE}/bin"
-HW_TYPE="$(uname -m)"
-PLATFORM_TYPE="$(uname)"
-SNAPSHOT="true"
-PYTEST_ADDOPTS=""
-
-SETUP_GVM_VERSION="v0.5.1"
-ASDF_MAGE_VERSION="1.14.0"
-SETUP_WIN_PYTHON_VERSION="3.11.0"
-
-# Docker & DockerHub
-DOCKER_COMPOSE_VERSION="1.21.0"
-DOCKER_REGISTRY="docker.elastic.co"
-
-ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET")
-PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET")
-GO_MOD_CHANGES=$(changeset_applies "^go.mod")
-
-KIND_VERSION="v0.20.0"
-KUBECONFIG="${WORKSPACE}/kubecfg"
-
-export WORKSPACE
-export BIN
-export HW_TYPE
-export PLATFORM_TYPE
-export SNAPSHOT
-export PYTEST_ADDOPTS
-
-export SETUP_GVM_VERSION
-export ASDF_MAGE_VERSION
-export SETUP_WIN_PYTHON_VERSION
-
-export DOCKER_COMPOSE_VERSION
-export DOCKER_REGISTRY
-
-export ONLY_DOCS
-export PACKAGING_CHANGES
-export GO_MOD_CHANGES
-
-export KIND_VERSION
-export KUBECONFIG
-
-add_bin_path
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 261f2bbee2d..68fd08a75df 100755
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -102,40 +102,3 @@ set_git_config() {
   git config user.name "${GITHUB_USERNAME_SECRET}"
   git config user.email "${GITHUB_EMAIL_SECRET}"
 }
-
-changeset_applies() {
-  local changeset=$1
-  if are_files_changed "$changeset"; then
-    echo true
-  else
-    echo false
-  fi
-}
-
-unset_secrets () {
-  for var in $(printenv | sed 's;=.*;;' | sort); do
-    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
-      unset "$var"
-    fi
-  done
-}
-
-google_cloud_logout_active_account() {
-  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
-  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
-    echo "Logging out from GCP for active account"
-    gcloud auth revoke $active_account > /dev/null 2>&1
-  else
-    echo "No active GCP accounts found."
-  fi
-  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
-    unset GOOGLE_APPLICATION_CREDENTIALS
-    cleanup
-  fi
-}
-
-cleanup() {
-  echo "Deleting temporary files..."
-  rm -rf ${BIN}/${TMP_FOLDER}.*
-  echo "Done."
-}
diff --git a/.buildkite/filebeat/scripts/packaging/package-util.sh b/.buildkite/filebeat/scripts/packaging/package-util.sh
deleted file mode 100755
index cb409e8d484..00000000000
--- a/.buildkite/filebeat/scripts/packaging/package-util.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-is_pr() {
-  if [[ $BUILDKITE_PULL_REQUEST != false ]]; then
-    return 0
-  else
-    return 1
-  fi
-}
-
-define_tags() {
-  aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
-  tags=("${BUILDKITE_COMMIT}")
-
-  if is_pr; then
-    tags+=("pr-${GITHUB_PR_NUMBER}")
-  else
-    tags+=("${SOURCE_TAG}" "${aliasVersion}")
-  fi
-}
-
-check_is_arm() {
-  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
-    is_arm="-arm"
-  else
-    is_arm=""
-  fi
-}
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
deleted file mode 100755
index f338e97c0b4..00000000000
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/linux-env.sh
-source .buildkite/filebeat/scripts/packaging/package-util.sh
-
-IMG_POSTFIX="-SNAPSHOT"
-VARIANTS=("" "-ubi" "-oss")
-VERSION="$(make get-version)"
-SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
-BEAT_NAME="filebeat"
-TARGET="observability-ci/${BEAT_NAME}"
-
-echo "--- Creating package"
-mage -d filebeat package
-
-#echo "--- Distribution list"
-#dir="filebeat/build/distributions"
-#buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
-#
-#echo "--- Docker image list"
-#docker images
-#
-#define_tags
-#check_is_arm
-#
-#echo "--- Tag & Push"
-#for variant in "${VARIANTS[@]}"; do
-#  source="beats/${BEAT_NAME}${variant}"
-#
-#  for tag in "${tags[@]}"; do
-#  targetTag=$tag${is_arm}
-#
-#  sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
-#  targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
-#  # Remove following lines once beats fully migrated
-#  targetName="${targetName}-buildkite"
-#
-#  if docker image inspect "${sourceName}" &>/dev/null; then
-#    echo "Source name: $sourceName Target name: $targetName"
-#    docker tag "$sourceName" "$targetName"
-##    docker push "$targetName"
-#
-#  else
-#    echo "Docker image ${sourceName} does not exist"
-#  fi
-#done
-#done
diff --git a/.buildkite/filebeat/scripts/packaging/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
deleted file mode 100755
index 02c942361b4..00000000000
--- a/.buildkite/filebeat/scripts/packaging/packaging-env.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-source .buildkite/env-scripts/util.sh
-
-docs_changeset="^.*\.(asciidoc|md)$
-deploy/kubernetes/.*-kubernetes.yaml"
-packaging_changeset="^dev-tools/packaging/
-^.go-version"
-
-VERSION=$(make get-version | tr -d '\n')
-ONLY_DOCS=$(changeset_applies "$docs_changeset")
-PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
-GO_MOD_CHANGES=$(changeset_applies "^go.mod")
-# Change the postfix to -SNAPSHOT, once Jenkins is disabled
-
-
-export PACKAGING_CHANGES
-export ONLY_DOCS
-export GO_MOD_CHANGES
-export DOCKER_REGISTRY
-export VERSION
-export REPO
-#export IMG_POSTFIX
-
-
-#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
-
-#IMAGES -> foreach = tag and push
-# registry: ${REGISTRY},
-# sourceTag: calculate_tags->sourceTag,
-# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
-# source: ${SOURCE},
-# target: ${TARGET}
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 7fe5cce11c1..fb5f881d878 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -30,81 +30,81 @@ steps:
     key: "heartbeat-mandatory-tests"
 
     steps:
-      - label: ":ubuntu: Heartbeat Unit Tests"
-        command: |
-          cd heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Ubuntu Unit Tests"
-
-      - label: ":rhel: Heartbeat Rhel9 Unit Tests"
-        command: |
-          cd heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_RHEL9}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Rhel9 Unit Tests"
-
-      - label: ":windows: Win-2016 Unit Tests"
-        key: "windows-2016"
-        command: |
-          Set-Location -Path heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2016}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Win-2016 Unit Tests"
-
-      - label: ":windows: Heartbeat Win-2022 Unit Test"
-        key: "windows-2022"
-        command: |
-          Set-Location -Path heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2022}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Win-2022 Unit Tests"
+#      - label: ":ubuntu: Heartbeat Unit Tests"
+#        command: |
+#          cd heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Ubuntu Unit Tests"
+#
+#      - label: ":rhel: Heartbeat Rhel9 Unit Tests"
+#        command: |
+#          cd heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_RHEL9}"
+#          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Rhel9 Unit Tests"
+#
+#      - label: ":windows: Win-2016 Unit Tests"
+#        key: "windows-2016"
+#        command: |
+#          Set-Location -Path heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2016}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Win-2016 Unit Tests"
+#
+#      - label: ":windows: Heartbeat Win-2022 Unit Test"
+#        key: "windows-2022"
+#        command: |
+#          Set-Location -Path heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2022}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Win-2022 Unit Tests"
 
       - label: ":ubuntu: Heartbeat Go (Module) Integration Tests"
         command: |
@@ -116,6 +116,10 @@ steps:
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet heartbeat
 
+          source .buildkite/scripts/define_module.sh
+          defineModule heartbeat
+          # echo "~~~ New MODULE definition is $$NEW_MODULE"
+
           echo "~~~ Running tests"
           cd heartbeat
           mage goIntegTest
@@ -133,213 +137,211 @@ steps:
           - github_commit_status:
               context: "heartbeat: Go Integration Tests"
 
-      - label: ":ubuntu: Heartbeat Python (Module) Integration Tests"
-        command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet heartbeat
-
-          echo "~~~ Running tests"
-          cd heartbeat
-          mage pythonIntegTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Python Integration Tests"
-
-  - group: "Heartbeat Extended Tests"
-    key: "heartbeat-extended-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
-
-    steps:
-      - label: ":linux: Heartbeat arm64 Unit Tests"
-        key: "arm-extended"
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command: |
-          cd heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths: "heartbeat/build/*.xml"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Linux arm64 Unit Tests"
-
-  - group: "Heartbeat Extended Testing MacOS"
-    key: "heartbeat-extended-tests-macos"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#      - label: ":ubuntu: Heartbeat Python (Module) Integration Tests"
+#        command: |
+#          set -euo pipefail
+#          # defines the MODULE env var based on what's changed in a PR
+#          source .buildkite/scripts/changesets.sh
+#          defineModuleFromTheChangeSet heartbeat
+#
+#          source .buildkite/scripts/define_module.sh
+#          defineModule heartbeat
+#          # echo "~~~ New MODULE definition is $$NEW_MODULE"
+#
+#          echo "~~~ Running tests"
+#          cd heartbeat
+#          mage pythonIntegTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Python Integration Tests"
 
-    steps:
-      - label: ":mac: Heartbeat MacOS Unit Tests"
-      - label: ":mac: macOS x86_64 Unit Tests"
-        key: "macos-extended"
-        if:  build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          cd heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: macOS x86_64 Unit Tests"
-
-      - label: ":mac: macOS arm64 Unit Tests"
-        key: "macos-arm-extended"
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          cd heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: macOS arm64 Unit Tests"
-
-  - group: "Heartbeat Windows Extended Testing"
-    key: "heartbeat-extended-tests-win"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
-
-    steps:
-      - label: ":windows: Win-2019 Unit Tests"
-        key: "heartbeat-win-extended-2019"
-        command: |
-          Set-Location -Path heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Win-2019 Unit Tests"
-
-      - label: ":windows: Win-11 Unit Tests"
-        key: "heartbeat-windows-extended-11"
-        command: |
-          Set-Location -Path heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_11}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Win-11 Unit Tests"
-
-      - label: ":windows: Win-10 Unit Tests"
-        key: "heartbeat-windows-extended-10"
-        command: |
-          Set-Location -Path heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_10}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "heartbeat/build/*.xml"
-          - "heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Win-10 Unit Tests"
-
-  - wait: ~
-    # with PRs, we want to run packaging only if mandatory tests succeed
-    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
-    # this allows building DRA artifacts even if there is flakiness in mandatory tests
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on: "heartbeat-mandatory-tests"
-
-  - group: "Heartbeat Packaging"
-    key: "heartbeat-packaging"
-    steps:
-      - label: ":ubuntu: Heartbeat Packaging Linux"
-        key: "heartbeat-package-linux"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-          SNAPSHOT: true
-        command: |
-          cd heartbeat
-          mage package
-        retry:
-          automatic:
-            - limit: 3
-        timeout_in_minutes: 20
-        agents:
-          provider: gcp
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Packaging Linux"
-
-      - label: ":linux: Heartbeat Packaging Linux arm64"
-        key: "heartbeat-package-linux-arm"
-        env:
-          PLATFORMS: "linux/arm64"
-          PACKAGES: "docker"
-          SNAPSHOT: true
-        command: |
-          cd heartbeat
-          mage package
-        retry:
-          automatic:
-            - limit: 3
-        timeout_in_minutes: 20
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        notify:
-          - github_commit_status:
-              context: "heartbeat: Packaging Linux arm64"
+#  - group: "Heartbeat Extended Tests"
+#    key: "heartbeat-extended-tests"
+#    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
+#
+#    steps:
+#      - label: ":linux: Heartbeat arm64 Unit Tests"
+#        key: "arm-extended"
+#        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+#        command: |
+#          cd heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "aws"
+#          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+#          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+#        artifact_paths: "heartbeat/build/*.xml"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Linux arm64 Unit Tests"
+#
+#      - label: ":mac: macOS x86_64 Unit Tests"
+#        key: "macos-extended"
+#        if:  build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#        command: |
+#          set -euo pipefail
+#          source .buildkite/scripts/install_macos_tools.sh
+#          cd heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_X86_64}"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: macOS x86_64 Unit Tests"
+#
+#      - label: ":mac: macOS arm64 Unit Tests"
+#        key: "macos-arm-extended"
+#        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#        command: |
+#          set -euo pipefail
+#          source .buildkite/scripts/install_macos_tools.sh
+#          cd heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_ARM}"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: macOS arm64 Unit Tests"
+#
+#  - group: "Heartbeat Windows Extended Testing"
+#    key: "heartbeat-extended-tests-win"
+#    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+#
+#    steps:
+#      - label: ":windows: Win-2019 Unit Tests"
+#        key: "heartbeat-win-extended-2019"
+#        command: |
+#          Set-Location -Path heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2019}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Win-2019 Unit Tests"
+#
+#      - label: ":windows: Win-11 Unit Tests"
+#        key: "heartbeat-windows-extended-11"
+#        command: |
+#          Set-Location -Path heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_11}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Win-11 Unit Tests"
+#
+#      - label: ":windows: Win-10 Unit Tests"
+#        key: "heartbeat-windows-extended-10"
+#        command: |
+#          Set-Location -Path heartbeat
+#          mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_10}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "heartbeat/build/*.xml"
+#          - "heartbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Win-10 Unit Tests"
+#
+#  - wait: ~
+#    # with PRs, we want to run packaging only if mandatory tests succeed
+#    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+#    # this allows building DRA artifacts even if there is flakiness in mandatory tests
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+#    depends_on: "heartbeat-mandatory-tests"
+#
+#  - group: "Heartbeat Packaging"
+#    key: "heartbeat-packaging"
+#    steps:
+#      - label: ":ubuntu: Heartbeat Packaging Linux"
+#        key: "heartbeat-package-linux"
+#        env:
+#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+#          SNAPSHOT: true
+#        command: |
+#          cd heartbeat
+#          mage package
+#        retry:
+#          automatic:
+#            - limit: 3
+#        timeout_in_minutes: 20
+#        agents:
+#          provider: gcp
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Packaging Linux"
+#
+#      - label: ":linux: Heartbeat Packaging Linux arm64"
+#        key: "heartbeat-package-linux-arm"
+#        env:
+#          PLATFORMS: "linux/arm64"
+#          PACKAGES: "docker"
+#          SNAPSHOT: true
+#        command: |
+#          cd heartbeat
+#          mage package
+#        retry:
+#          automatic:
+#            - limit: 3
+#        timeout_in_minutes: 20
+#        agents:
+#          provider: "aws"
+#          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+#          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+#        notify:
+#          - github_commit_status:
+#              context: "heartbeat: Packaging Linux arm64"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 9a88715cded..428b7f84b53 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/util.sh
-
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
@@ -11,53 +9,6 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then
-  source .buildkite/env-scripts/env.sh
-  if [[ -z "${GO_VERSION-""}" ]]; then
-    export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
-  fi
-
-  # Images with prefix "platform-ingest-beats-*" has Go pre-setup.
-  # Image itself takes care of Go version download/installation based on .go-version file
-  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
-    if [[ -z "${GO_VERSION-""}" ]]; then
-      GO_VERSION=$(cat "${WORKSPACE}/.go-version")
-      export GO_VERSION
-    fi
-
-    # To prevent "OSError: [Errno 24] Too many open files"
-    ulimit -Sn 150000
-    echo "--- Ulimit: $(ulimit)"
-
-    echo "--- Setting up environment"
-    add_bin_path
-    with_go
-    with_mage
-  fi
-
-  if [[ "$BUILDKITE_STEP_KEY" == *packaging* ]]; then
-    DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
-
-    export DOCKER_USERNAME_SECRET
-    export DOCKER_PASSWORD_SECRET
-    export GITHUB_TOKEN_SECRET
-
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
-
-    github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
-    github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
-
-    git config user.name "$github_username"
-    git config user.email "$github_email"
-  fi
-
-  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
-    ulimit -Sn 30000
-  fi
-fi
-
 ENABLED_BEATS_PIPELINES_SLUGS=(
   "auditbeat"
   "heartbeat"
diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
index ed9fb14f3d4..a91a2697646 100644
--- a/.buildkite/metricbeat/pipeline.yml
+++ b/.buildkite/metricbeat/pipeline.yml
@@ -33,75 +33,82 @@ steps:
   - group: "Metricbeat Mandatory Tests"
     key: "metricbeat-mandatory-tests"
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
-        key: "mandatory-linux-unit-test"
-        command: "cd metricbeat && mage build unitTest"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Ubuntu Unit Tests"
+#      - label: ":linux: Ubuntu Unit Tests"
+#        key: "mandatory-linux-unit-test"
+#        command: "cd metricbeat && mage build unitTest"
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Ubuntu Unit Tests"
+#
+#      - label: ":go: Go Integration Tests"
+#        key: "mandatory-int-test"
+#        command: |
+#          set -euo pipefail
+#          # defines the MODULE env var based on what's changed in a PR
+#          source .buildkite/scripts/changesets.sh
+#          defineModuleFromTheChangeSet metricbeat
+#          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+#
+#          # TODO move this section to base image / pre-command hook
+#          echo "~~~ Installing kind"
+#          asdf plugin add kind
+#          asdf install kind $ASDF_KIND_VERSION
+#
+#          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
+#
+#          echo "~~~ Running tests"
+#          export KUBECONFIG="$$PWD/kubecfg"
+#          cd metricbeat && mage goIntegTest
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Go Integration Tests"
 
-      - label: ":go: Go Integration Tests"
-        key: "mandatory-int-test"
+      - label: ":python: Python Integration Tests"
+        key: "mandatory-python-int-test"
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Running tests"
+          echo "~~~ OLD MODULE definition is $$MODULE"
 
-          # TODO move this section to base image / pre-command hook
-          echo "~~~ Installing kind"
-          asdf plugin add kind
-          asdf install kind $ASDF_KIND_VERSION
+          source .buildkite/scripts/define_module.sh
+          defineModule metricbeat
+          echo "~~~ New MODULE definition is $$MODULE"
 
-          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
+          MODULE=$(go run .buildkite/scripts/moduleDefinition/main.go metricbeat)
 
-          echo "~~~ Running tests"
-          export KUBECONFIG="$$PWD/kubecfg"
-          cd metricbeat && mage goIntegTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Go Integration Tests"
+          export MODULE
+          echo "~~~ GO MODULE definition is $$MODULE"
 
-      - label: ":python: Python Integration Tests"
-        key: "mandatory-python-int-test"
-        command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Running tests"
-
-          # TODO move this section to base image / pre-command hook
           echo "~~~ Installing kind"
           asdf plugin add kind
           asdf install kind $ASDF_KIND_VERSION
-
           .buildkite/deploy/kubernetes/scripts/kind-setup.sh
 
-          echo "~~~ Running tests"
-          export KUBECONFIG="$$PWD/kubecfg"
-          cd metricbeat && mage pythonIntegTest
+#          echo "~~~ Running tests"
+#          export KUBECONFIG="$$PWD/kubecfg"
+#          cd metricbeat && mage pythonIntegTest
         retry:
           automatic:
             - limit: 3
@@ -116,218 +123,219 @@ steps:
           - github_commit_status:
               context: "metricbeat: Python Integration Tests"
 
-      - label: ":negative_squared_cross_mark: Cross compile"
-        key: "mandatory-cross-compile"
-        command: "make -C metricbeat crosscompile"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Cross compile"
-
-      - label: ":windows: Windows 2016 Unit Tests"
-        command: |
-          Set-Location -Path metricbeat
-          mage build unitTest
-        key: "mandatory-win-2016-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2016}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Windows 2016 Unit Tests"
-
-      - label: ":windows: Windows 2022 Unit Tests"
-        command: |
-          Set-Location -Path metricbeat
-          mage build unitTest
-        key: "mandatory-win-2022-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2022}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Windows 2022 Unit Tests"
-
-  - group: "Metricbeat Extended Windows Tests"
-    key: "metricbeat-extended-win-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
-    steps:
-      - label: ":windows: Windows 10 Unit Tests"
-        command: |
-          Set-Location -Path metricbeat
-          mage build unitTest
-        key: "extended-win-10-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_10}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Extended Windows 10 Unit Tests"
-
-      - label: ":windows: Windows 11 Unit Tests"
-        command: |
-          Set-Location -Path metricbeat
-          mage build unitTest
-        key: "extended-win-11-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_11}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Extended Windows 11 Unit Tests"
-
-      - label: ":windows: Windows 2019 Unit Tests"
-        command: |
-          Set-Location -Path metricbeat
-          mage build unitTest
-        key: "extended-win-2019-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Extended Windows 2019 Unit Tests"
-
-  - group: "Metricbeat Extended MacOS Tests"
-    key: "metricbeat-extended-macos-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-    steps:
-      - label: ":mac: MacOS x64_64 Unit Tests"
-        key: "extended-macos-x64-64-unit-tests"
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          cd metricbeat && mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Extended MacOS x86_64 Unit Tests"
-
-      - label: ":mac: MacOS arm64 Unit Tests"
-        key: "extended-macos-arm64-unit-tests"
-        skip: "due to https://github.com/elastic/beats/issues/33035"
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          cd metricbeat && mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths:
-          - "metricbeat/build/*.xml"
-          - "metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Extended MacOS arm64 Unit Tests"
-
-  - wait: ~
-    # with PRs, we want to run packaging only if mandatory tests succeed
-    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
-    # this allows building DRA artifacts even if there is flakiness in mandatory tests
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on: "metricbeat-mandatory-tests"
-
-  - group: "Metricbeat Packaging"
-    key: "metricbeat-packaging"
-    steps:
-      - label: ":linux: Packaging Linux"
-        key: "packaging-linux"
-        command: "cd metricbeat && mage package"
-        retry:
-          automatic:
-            - limit: 3
-        timeout_in_minutes: 20
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Packaging Linux"
-
-      - label: ":linux: Packaging ARM"
-        key: "packaging-arm"
-        command: "cd metricbeat && mage package"
-        retry:
-          automatic:
-            - limit: 3
-        timeout_in_minutes: 20
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        env:
-          PLATFORMS: "linux/arm64"
-          PACKAGES: "docker"
-        notify:
-          - github_commit_status:
-              context: "metricbeat: Packaging Linux ARM"
+#      - label: ":negative_squared_cross_mark: Cross compile"
+#        key: "mandatory-cross-compile"
+#        command: "make -C metricbeat crosscompile"
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Cross compile"
+#
+#      - label: ":windows: Windows 2016 Unit Tests"
+#        command: |
+#          Set-Location -Path metricbeat
+#          mage build unitTest
+#        key: "mandatory-win-2016-unit-tests"
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2016}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Windows 2016 Unit Tests"
+#
+#      - label: ":windows: Windows 2022 Unit Tests"
+#        command: |
+#          Set-Location -Path metricbeat
+#          mage build unitTest
+#        key: "mandatory-win-2022-unit-tests"
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2022}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Windows 2022 Unit Tests"
+#
+#  - group: "Metricbeat Extended Windows Tests"
+#    key: "metricbeat-extended-win-tests"
+#    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+#    steps:
+#      - label: ":windows: Windows 10 Unit Tests"
+#        command: |
+#          Set-Location -Path metricbeat
+#          mage build unitTest
+#        key: "extended-win-10-unit-tests"
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_10}"
+#          machineType: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Extended Windows 10 Unit Tests"
+#
+#      - label: ":windows: Windows 11 Unit Tests"
+#        command: |
+#          Set-Location -Path metricbeat
+#          mage build unitTest
+#        key: "extended-win-11-unit-tests"
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_11}"
+#          machineType: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Extended Windows 11 Unit Tests"
+#
+#      - label: ":windows: Windows 2019 Unit Tests"
+#        command: |
+#          Set-Location -Path metricbeat
+#          mage build unitTest
+#        key: "extended-win-2019-unit-tests"
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2019}"
+#          machineType: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Extended Windows 2019 Unit Tests"
+#
+#  - group: "Metricbeat Extended MacOS Tests"
+#    key: "metricbeat-extended-macos-tests"
+#    if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#    steps:
+#      - label: ":mac: MacOS x64_64 Unit Tests"
+#        key: "extended-macos-x64-64-unit-tests"
+#        command: |
+#          set -euo pipefail
+#          source .buildkite/scripts/install_macos_tools.sh
+#          cd metricbeat && mage build unitTest
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_X86_64}"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Extended MacOS x86_64 Unit Tests"
+#
+#      - label: ":mac: MacOS arm64 Unit Tests"
+#        key: "extended-macos-arm64-unit-tests"
+#        skip: "due to https://github.com/elastic/beats/issues/33035"
+#        command: |
+#          set -euo pipefail
+#          source .buildkite/scripts/install_macos_tools.sh
+#          cd metricbeat && mage build unitTest
+#        retry:
+#          automatic:
+#           - limit: 3
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_ARM}"
+#        artifact_paths:
+#          - "metricbeat/build/*.xml"
+#          - "metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Extended MacOS arm64 Unit Tests"
+#
+#
+#  - wait: ~
+#    # with PRs, we want to run packaging only if mandatory tests succeed
+#    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+#    # this allows building DRA artifacts even if there is flakiness in mandatory tests
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+#    depends_on: "metricbeat-mandatory-tests"
+#
+#  - group: "Metricbeat Packaging"
+#    key: "metricbeat-packaging"
+#    steps:
+#      - label: ":linux: Packaging Linux"
+#        key: "packaging-linux"
+#        command: "cd metricbeat && mage package"
+#        retry:
+#          automatic:
+#            - limit: 3
+#        timeout_in_minutes: 20
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        env:
+#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Packaging Linux"
+#
+#      - label: ":linux: Packaging ARM"
+#        key: "packaging-arm"
+#        command: "cd metricbeat && mage package"
+#        retry:
+#          automatic:
+#            - limit: 3
+#        timeout_in_minutes: 20
+#        agents:
+#          provider: "aws"
+#          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+#          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+#        env:
+#          PLATFORMS: "linux/arm64"
+#          PACKAGES: "docker"
+#        notify:
+#          - github_commit_status:
+#              context: "metricbeat: Packaging Linux ARM"
diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh
index 5c6fd6c7b0d..de7cad8e710 100644
--- a/.buildkite/scripts/changesets.sh
+++ b/.buildkite/scripts/changesets.sh
@@ -69,9 +69,11 @@ defineModuleFromTheChangeSet() {
   done
 
   # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993
-  if [[ ! -z $changed_modules ]]; then 
+  if [[ ! -z $changed_modules ]]; then
     export MODULE="${changed_modules}"
     echo "~~~ Set env var MODULE to [$MODULE]"
     echo "~~~ Resuming commands"
+  else
+    export MODULE="null"
   fi
 }
diff --git a/.buildkite/scripts/define_module.sh b/.buildkite/scripts/define_module.sh
index d7187b58901..6134607f82b 100755
--- a/.buildkite/scripts/define_module.sh
+++ b/.buildkite/scripts/define_module.sh
@@ -4,52 +4,59 @@ set -euo pipefail
 
 OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
-BEAT_PATH=$1
-MODULE=''
 
 definePattern() {
-  pattern="${XPACK_MODULE_PATTERN}"
+  pattern="${OSS_MODULE_PATTERN}"
 
-  if [[ "$BEAT_PATH" == *"x-pack/"* ]]; then
-    pattern="${OSS_MODULE_PATTERN}"
+  if [[ "$beatPath" == *"x-pack/"* ]]; then
+    pattern="${XPACK_MODULE_PATTERN}"
   fi
 }
 
 defineExclusions() {
-  local transformedDirectory=${BEAT_PATH//\//\\\/}
-  local exclusion="((?!^${transformedDirectory}\\/).)*\$"
-  exclude="^(${exclusion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)"
+  exclude="^$beatPath\/module\/(.*(?<!\.asciidoc|\.png))$"
 }
 
-getGitMatchingGroup() {
+defineFromCommit() {
   local previousCommit
-  local matches
-  local match
-
   local changeTarget=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-$BUILDKITE_BRANCH}
-  local baseCommit=$BUILDKITE_COMMIT
+
   previousCommit=$(git rev-parse HEAD^)
-  from=$(echo ${changeTarget:+"origin/$changeTarget"}${previousCommit:-$baseCommit})
-  to=$baseCommit
 
-  matches=$(git diff --name-only "$from"..."$to" | grep -v "$exclude" | grep -oP "$pattern" | sort -u)
+  from=${changeTarget:+"origin/$changeTarget"}
+  from=${from:-$previousCommit}
+  from=${from:-$BUILDKITE_COMMIT}
+}
 
-  match=$(echo "$matches" | wc -w)
+getMatchingModules() {
+  local changedPaths
+  mapfile -t changedPaths < <(git diff --name-only "$from"..."$BUILDKITE_COMMIT" | grep -P "$exclude" | grep -oP "$pattern")
+  mapfile -t modulesMatched < <(printf "%s\n" "${changedPaths[@]}" | grep -o 'module/.*' | awk -F '/' '{print $2}' | sort -u)
+}
 
-  if [ "$match" -eq 1 ]; then
-    echo "$matches"
-  else
-    echo ''
-  fi
+addToModuleEnvVar() {
+  local module
+  for module in "${modulesMatched[@]}"; do
+    if [[ -z ${modules+x} ]]; then
+      modules="$module"
+    else
+      modules+=",$module"
+    fi
+  done
 }
 
 defineModule() {
-  cd "${BEAT_PATH}"
-  NEW_MODULE=$(getGitMatchingGroup "$pattern" "$exclude")
-  if [ ! -f "$BEAT_PATH/module/${module}" ]; then
-    NEW_MODULE=''
-  fi
-  cd - >/dev/null
+  beatPath=$1
+
+  definePattern
+  defineExclusions
+  defineFromCommit
+  getMatchingModules
 
-  export NEW_MODULE
+  if [ "${#modulesMatched[@]}" -gt 0 ]; then
+    addToModuleEnvVar
+    export MODULE=$modules
+  else
+    echo "~~~ No changes in modules for $beatPath"
+  fi
 }
diff --git a/.buildkite/scripts/moduleDefinition/go.mod b/.buildkite/scripts/moduleDefinition/go.mod
new file mode 100644
index 00000000000..5eaf5ca38b1
--- /dev/null
+++ b/.buildkite/scripts/moduleDefinition/go.mod
@@ -0,0 +1,3 @@
+module github.com/elastic/beats/module
+
+go 1.21.9
diff --git a/.buildkite/scripts/moduleDefinition/main.go b/.buildkite/scripts/moduleDefinition/main.go
new file mode 100755
index 00000000000..0a5d25f39f5
--- /dev/null
+++ b/.buildkite/scripts/moduleDefinition/main.go
@@ -0,0 +1,102 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("Argument required: beatPath")
+		os.Exit(1)
+	}
+
+	beatPath := os.Args[1]
+
+	var modulePattern string
+	if strings.Contains(beatPath, "x-pack") {
+		modulePattern = "^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+	} else {
+		modulePattern = "^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+	}
+
+	moduleRegex, err := regexp.Compile(modulePattern)
+	if err != nil {
+		log.Fatal("failed to compile regex: " + err.Error())
+	}
+
+	modules := make(map[string]bool)
+	for _, line := range getDiff() {
+		if !isAsciiOrPng(line) {
+			matches := moduleRegex.FindStringSubmatch(line)
+			if len(matches) > 0 {
+				modules[matches[1]] = true
+			}
+		}
+	}
+
+	keys := make([]string, len(modules))
+	i := 0
+	for k := range modules {
+		keys[i] = k
+		i++
+	}
+
+	fmt.Println(strings.Join(keys, ","))
+}
+
+func isAsciiOrPng(file string) bool {
+	return strings.HasSuffix(file, ".asciidoc") || strings.HasSuffix(file, ".png")
+}
+
+func getDiff() []string {
+	cmd := exec.Command("git", "diff", "--name-only", getCommitRange())
+	output, err := cmd.Output()
+	if err != nil {
+		fmt.Printf("Error: %s", err)
+		os.Exit(1)
+	}
+
+	return strings.Split(string(output), "\n")
+}
+
+func getFromCommit() string {
+	baseBranch := os.Getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH")
+	branch := os.Getenv("BUILDKITE_BRANCH")
+	commit := os.Getenv("BUILDKITE_COMMIT")
+
+	if baseBranch != "" {
+		return fmt.Sprintf("origin/%s", baseBranch)
+	}
+
+	if branch != "" {
+		return fmt.Sprintf("origin/%s", branch)
+	}
+
+	previousCommit := getPreviousCommit()
+	if previousCommit != "" {
+		return previousCommit
+	}
+
+	return commit
+}
+
+func getPreviousCommit() string {
+	cmd := exec.Command("git", "rev-parse", "HEAD^")
+	output, err := cmd.Output()
+	if err != nil {
+		fmt.Println("Error: ", err)
+		os.Exit(1)
+	}
+
+	return string(output)
+}
+
+func getCommitRange() string {
+	commit := os.Getenv("BUILDKITE_COMMIT")
+	return fmt.Sprintf("%s...%s", getFromCommit(), commit)
+}
diff --git a/metricbeat/module/linux/deleteme.asciidoc b/metricbeat/module/linux/deleteme.asciidoc
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/metricbeat/module/linux/deleteme.go b/metricbeat/module/linux/deleteme.go
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/metricbeat/module/linux/deleteme.png b/metricbeat/module/linux/deleteme.png
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/metricbeat/module/linux/deleteme1/deletemeagain.go b/metricbeat/module/linux/deleteme1/deletemeagain.go
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go
index 5865867cf1f..db430268b2d 100644
--- a/metricbeat/module/logstash/logstash_integration_test.go
+++ b/metricbeat/module/logstash/logstash_integration_test.go
@@ -63,6 +63,7 @@ func TestFetch(t *testing.T) {
 
 			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
 				events[0].BeatEvent("logstash", metricSet).Fields.StringToPrint())
+			t.Logf("%s just to test changesets", f.Name())
 		})
 	}
 }
diff --git a/x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go b/x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go
new file mode 100644
index 00000000000..30bc49fa39e
--- /dev/null
+++ b/x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go
@@ -0,0 +1 @@
+package subdir
diff --git a/x-pack/auditbeat/module/system/deleteme.go b/x-pack/auditbeat/module/system/deleteme.go
new file mode 100644
index 00000000000..9b140a34dd3
--- /dev/null
+++ b/x-pack/auditbeat/module/system/deleteme.go
@@ -0,0 +1 @@
+package system
diff --git a/x-pack/metricbeat/module/deleteme.go b/x-pack/metricbeat/module/deleteme.go
new file mode 100644
index 00000000000..b0b78bfd7a2
--- /dev/null
+++ b/x-pack/metricbeat/module/deleteme.go
@@ -0,0 +1 @@
+package module
diff --git a/x-pack/metricbeat/module/deleteme/deletme.go b/x-pack/metricbeat/module/deleteme/deletme.go
new file mode 100644
index 00000000000..935f8ef80cd
--- /dev/null
+++ b/x-pack/metricbeat/module/deleteme/deletme.go
@@ -0,0 +1 @@
+package deleteme
diff --git a/x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go b/x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go
new file mode 100644
index 00000000000..30bc49fa39e
--- /dev/null
+++ b/x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go
@@ -0,0 +1 @@
+package subdir