Buildkite pipeline unification fixes (#38871)

* test - added empty file

* removed test dummy file

* restored snapshot packaging step

* dummy file to test packaging

* pr fixes

* fixing packaging step

* fixing packaging step

* moved xpack dockerlogbeat to main pipeline

* restored snapshot packaging step

* fixing packaging step

* moved xpack-winlogbeat to main pipeline

* general pipeline fixes

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -4,24 +4,20 @@ env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
 
-notify:
-  - github_commit_status:
-      context: "Deploy/K8S"
-
 steps:
-  - group: "Depoy/K8S"
+  - group: "Deploy/K8S"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
 
     steps:
     - label: "Checks"
       command: ".buildkite/deploy/kubernetes/scripts/make.sh"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-checks"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-checks"
 
     - label: "K8S Test/K8S version: v1.29.0"
       key: "k8s-test-129"
@@ -30,13 +26,13 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.29.0"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.29.0"
 
     - label: "K8S Test/K8S version: v1.28.0"
       key: "k8s-test-128"
@@ -45,13 +41,13 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.28.0"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.28.0"
 
     - label: "K8S Test/K8S version: v1.27.3"
       key: "k8s-test-1273"
@@ -60,13 +56,13 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.27.3"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.27.3"
 
     - label: "K8S Test/K8S version: v1.26.6"
       key: "k8s-test-1266"
@@ -75,10 +71,10 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.26.6"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.26.6"

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -1,10 +1,6 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  BEATS_PROJECT_NAME: "heartbeat"
-
-  ASDF_MAGE_VERSION: 1.15.0
-
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
@@ -22,20 +18,15 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
-  #Packaging
-  PACKAGING_ARM_PLATFORMS: "linux/arm64"
-  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-
-notify:
-  - github_commit_status:
-      context: "$BEATS_PROJECT_NAME"
+  # Other deps
+  ASDF_MAGE_VERSION: 1.15.0
 
 steps:
   - group: "Heartbeat Mandatory Testing"
     key: "heartbeat-mandatory-tests"
     steps:
       - label: ":ubuntu: Unit Tests"
-        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        command: "cd heartbeat && mage build unitTest"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -44,8 +35,8 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":rhel:-9 Unit Tests"
-        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+      - label: ":rhel: Heartbeat Rhel9 Unit Tests"
+        command: "cd heartbeat && mage build unitTest"
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -54,10 +45,10 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":windows:-2016 Unit Test"
+      - label: ":windows: Heartbeat Win-2016 Unit Test"
         key: "windows-2016"
         command: |
-          Set-Location -Path $BEATS_PROJECT_NAME
+          Set-Location -Path heartbeat
           mage build unitTest
         agents:
           provider: "gcp"
@@ -68,10 +59,10 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":windows:-2022 Unit Test"
+      - label: ":windows: Heartbeat Win-2022 Unit Test"
         key: "windows-2022"
         command: |
-          Set-Location -Path $BEATS_PROJECT_NAME
+          Set-Location -Path heartbeat
           mage build unitTest
         agents:
           provider: "gcp"
@@ -82,8 +73,8 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":ubuntu: Go Integration Tests"
-        command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest"
+      - label: ":ubuntu: Heartbeat Go Integration Tests"
+        command: "cd heartbeat && mage goIntegTest"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -92,8 +83,8 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":ubuntu: Python Integration Tests"
-        command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest"
+      - label: ":ubuntu: Heartbeat Python Integration Tests"
+        command: "cd heartbeat && mage pythonIntegTest"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -106,10 +97,10 @@ steps:
     key: "heartbeat-extended-tests-arm"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
     steps:
-      - label: ":linux: ARM64 Unit Tests"
+      - label: ":linux: Heartbeat ARM64 Unit Tests"
         key: "arm-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        command: "cd heartbeat && mage build unitTest"
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -120,7 +111,7 @@ steps:
     key: "heartbeat-extended-tests-macos"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
     steps:
-      - label: ":mac: MacOS Unit Tests"
+      - label: ":mac: Heartbeat MacOS Unit Tests"
         key: "macos-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest"
@@ -131,10 +122,10 @@ steps:
 
   - group: "Heartbeat Windows Extended Testing"
     key: "heartbeat-extended-tests-win"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
 
     steps:
-      - label: ":windows: Win 2019 Unit Tests"
+      - label: ":windows: Heartbeat Win-2019 Unit Tests"
         key: "heartbeat-win-extended-2019"
         command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         agents:
@@ -146,10 +137,10 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":windows:-11 Unit Tests"
+      - label: ":windows: Heartbeat Win-11 Unit Tests"
         key: "heartbeat-windows-extended-11"
         command: |
-          Set-Location -Path $BEATS_PROJECT_NAME
+          Set-Location -Path heartbeat
           mage build unitTest
         agents:
           provider: "gcp"
@@ -160,10 +151,10 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":windows:-10 Unit Tests"
+      - label: ":windows: Heartbeat Win-10 Unit Tests"
         key: "heartbeat-windows-extended-10"
         command: |
-          Set-Location -Path $BEATS_PROJECT_NAME
+          Set-Location -Path heartbeat
           mage build unitTest
         agents:
           provider: "gcp"
@@ -180,24 +171,24 @@ steps:
     depends_on:
       - "heartbeat-mandatory-tests"
     steps:
-      - label: ":ubuntu: Packaging Linux X86"
+      - label: ":ubuntu: Heartbeat Packaging Linux X86"
         key: "heartbeat-package-linux-x86"
         env:
-          PLATFORMS: "${PACKAGING_PLATFORMS}"
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
-        command: "cd $BEATS_PROJECT_NAME && mage package"
+        command: "cd heartbeat && mage package"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-      - label: ":linux: Packaging Linux ARM"
+      - label: ":linux: Heartbeat Packaging Linux ARM"
         key: "heartbeat-package-linux-arm"
         env:
-          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
+          PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
-        command: "cd $BEATS_PROJECT_NAME && mage package"
+        command: "cd heartbeat && mage package"
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"

.buildkite/pipeline.yml

@@ -36,6 +36,7 @@ steps:
                 - heartbeat/
                 - .buildkite/heartbeat/
                 - .buildkite/scripts
+                - .buildkite/hooks/
                 #OSS
                 - go.mod
                 - pytest.ini
@@ -87,6 +88,7 @@ steps:
             - path:
                 - x-pack/dockerlogbeat/
                 - .buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
+                - .buildkite/hooks/
                 - .buildkite/scripts
                 #OSS
                 - go.mod
@@ -113,6 +115,7 @@ steps:
                 - metricbeat/
                 - .buildkite/metricbeat/
                 - .buildkite/scripts
+                - .buildkite/hooks/
                 #OSS
                 - go.mod
                 - pytest.ini
@@ -165,6 +168,7 @@ steps:
                 - x-pack/winlogbeat/
                 - .buildkite/x-pack/pipeline.xpack.winlogbeat.yml
                 - .buildkite/scripts
+                - .buildkite/hooks/
                 #OSS
                 - go.mod
                 - pytest.ini
@@ -188,9 +192,10 @@ steps:
           watch:
             - path:
                 - .buildkite/deploy/kubernetes/**
+                - .buildkite/hooks/
+                - .buildkite/scripts
                 - deploy/kubernetes/**
                 - libbeat/docs/version.asciidoc
-                - .buildkite/scripts
               config:
                 trigger: "deploy-k8s"
                 build: