[7.17](backport #39239) Update x-pack/heartbeat pipeline to match Jen…

…kins (#39421)

* Update x-pack/heartbeat pipeline to match Jenkins (#39239)

* updated win-2016 test cmd

* updated macos namimg

* pr fixes

* removed pipeline masking where irrelevant

(cherry picked from commit 4f5b23a)

# Conflicts:
#	.buildkite/x-pack/pipeline.xpack.heartbeat.yml

* fixed conflicts

---------

Co-authored-by: Olga Naydyonock <[email protected]>

.buildkite/x-pack/pipeline.xpack.heartbeat.yml

@@ -2,7 +2,8 @@
 name: "beats-xpack-heartbeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
@@ -11,7 +12,6 @@ env:
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -30,7 +30,7 @@ steps:
   - group: "x-pack/heartbeat Mandatory Tests"
     key: "x-pack-heartbeat-mandatory-tests"
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
+      - label: ":ubuntu: x-pack/heartbeat Ubuntu Unit Tests"
         key: "mandatory-linux-unit-test"
         command: |
           set -euo pipefail
@@ -53,7 +53,7 @@ steps:
           - github_commit_status:
               context: "x-pack/heartbeat: Ubuntu Unit Tests"
 
-      - label: ":go: Go Integration Tests"
+      - label: ":ubuntu: x-pack/heartbeat Go Integration Tests"
         key: "mandatory-int-test"
         command: |
           set -euo pipefail
@@ -76,15 +76,15 @@ steps:
           - github_commit_status:
               context: "x-pack/heartbeat: Go Integration Tests"
 
-      - label: ":windows: Windows 2019 Unit Tests"
+      - label: ":windows: x-pack/heartbeat Windows 2019 Unit Tests"
         skip: "skipping due to elastic/beats#23957 and elastic/beats#23958"
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build test
-        key: "extended-win-2019-unit-tests"
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
+        key: "extended-win-2019-unit-tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -102,57 +102,17 @@ steps:
     key: "x-pack-heartbeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
     skip: "skipping due to elastic/beats#23957 and elastic/beats#23958"
-    steps:
-      - label: ":windows: Windows 10 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/heartbeat
-          mage build test
-        key: "extended-win-10-unit-tests"
-        retry:
-          automatic:
-           - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_10}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/heartbeat/build/*.xml"
-          - "x-pack/heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/heartbeat: Windows 10 Unit Tests"
-
-      - label: ":windows: Windows 11 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/heartbeat
-          mage build test
-        key: "extended-win-11-unit-tests"
-        retry:
-          automatic:
-           - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_11}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/heartbeat/build/*.xml"
-          - "x-pack/heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/heartbeat: Windows 11 Unit Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+    steps:
+      - label: ":windows: x-pack/heartbeat Windows 2016 Unit Tests"
         key: "mandatory-win-2016-unit-tests"
+        skip: "skipping due to elastic/beats#23957 and elastic/beats#23958"
         command: |
           Set-Location -Path x-pack/heartbeat
-          mage build unitTest
+          mage build test
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -166,32 +126,33 @@ steps:
           - github_commit_status:
               context: "x-pack/heartbeat: Windows 2016 Unit Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
-        key: "mandatory-win-2022-unit-tests"
+      - label: ":windows: x-pack/heartbeat Windows 10 Unit Tests"
         command: |
           Set-Location -Path x-pack/heartbeat
-          mage build unitTest
+          mage build test
+        key: "extended-win-10-unit-tests"
         retry:
           automatic:
            - limit: 3
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_2022}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          image: "${IMAGE_WIN_10}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
         artifact_paths:
           - "x-pack/heartbeat/build/*.xml"
           - "x-pack/heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/heartbeat: Windows 2022 Unit Tests"
-  
-  - group: "x-pack/heartbeat MacOS Extended Tests"
+              context: "x-pack/heartbeat: Windows 10 Unit Tests"
+
+  - group: "x-pack/heartbeat macOS Extended Tests"
     key: "x-pack-heartbeat-extended-tests-macos"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+
     steps:
-      - label: ":mac: MacOS x86_64 Unit Tests"
+      - label: ":mac: x-pack/heartbeat macOS x86_64 Unit Tests"
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -210,39 +171,21 @@ steps:
           - "x-pack/heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/heartbeat: MacOS x86_64 Extended Tests"
-
-      - label: ":mac: MacOS arm64 Unit Tests"
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          withNodeJSEnv $ASDF_NODEJS_VERSION
-          installNodeJsDependencies
-          cd x-pack/heartbeat
-          mage build unitTest
-        retry:
-          automatic:
-           - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths:
-          - "x-pack/heartbeat/build/*.xml"
-          - "x-pack/heartbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/heartbeat: MacOS arm64 Extended Tests"
+              context: "x-pack/heartbeat: macOS x86_64 Extended Tests"
 
   - wait: ~
+    # with PRs, we want to run packaging only if mandatory tests succeed
+    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in mandatory tests
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     depends_on:
       - step: "x-pack-heartbeat-mandatory-tests"
 
-  - group: "Packaging"
-    key: "packaging"
+  - group: "x-pack/heartbeat Packaging"
+    key: "x-pack-heartbeat-packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":ubuntu: x-pack/heartbeat Packaging Linux x86_64"
         key: "packaging-linux"
         command: |
           cd x-pack/heartbeat
@@ -257,20 +200,20 @@ steps:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
         notify:
           - github_commit_status:
-              context: "x-pack/heartbeat: Packaging Linux"
+              context: "x-pack/heartbeat: Packaging Linux x86_64"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":ubuntu: x-pack/heartbeat Packaging Linux arm64"
         key: "packaging-arm"
         command: |
           cd x-pack/heartbeat
           mage package
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "x-pack/heartbeat: Packaging Linux ARM"
+              context: "x-pack/heartbeat: Packaging Linux arm64"