diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index 8d50446a089..581fade27e0 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -53,6 +53,9 @@ steps:
         key: "mandatory-int-test"
         command: |
           set -euo pipefail
+          # docker-compose problem debugging
+          which docker-compose
+          docker-compose --version
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
@@ -72,226 +75,225 @@ steps:
           - github_commit_status:
               context: "x-pack/metricbeat: Go (MODULE) Integration Tests"
 
-      - label: ":ubuntu: x-pack/metricbeat Python (MODULE) Integration Tests"
-        key: "mandatory-python-int-test"
-        command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests"
-          cd x-pack/metricbeat && mage pythonIntegTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Python (MODULE) Integration Tests"
-
-      - label: ":windows: x-pack/metricbeat Win-2019 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/metricbeat
-          mage build unitTest
-        key: "extended-win-2019-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Win-2019 Unit Tests"
-
-  - group: "x-pack/metricbeat Extended Windows Tests"
-    key: "x-pack-metricbeat-extended-win-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
-
-    steps:
-      - label: ":windows: x-pack/metricbeat Win-2016 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/metricbeat
-          mage build unitTest
-        key: "mandatory-win-2016-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2016}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Win-2016 Unit Tests"
-
-      - label: ":windows: x-pack/metricbeat Win 10 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/metricbeat
-          mage build unitTest
-        key: "extended-win-10-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_10}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Win 10 Unit Tests"
-
-  - group: "x-pack/metricbeat Extended Tests"
-    key: "x-pack-metricbeat-extended-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/
-
-    steps:
-      - label: ":mac: x-pack/metricbeat macOS x86_64 Unit Tests"
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          cd x-pack/metricbeat && mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: macOS x86_64 Unit Tests"
-
-      - label: ":ubuntu: x-pack/metricbeat Cloud (MODULE) Tests"
-        key: "x-pack-metricbeat-extended-cloud-test"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
-        command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/metricbeat
-          source .buildkite/scripts/initCloudEnv.sh
-          echo "~~~ Running tests"
-          cd x-pack/metricbeat
-          mage build test
-        env:
-          ASDF_TERRAFORM_VERSION: 1.0.2
-          MODULE_DIR: "x-pack/metricbeat/module/aws"
-
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Cloud (MODULE) Tests"
-
-      - label: ":ubuntu: x-pack/metricbeat Cloud AWS (MODULE) Tests"
-        key: "x-pack-metricbeat-extended-cloud-test-aws"
-#        skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3399"
-        # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3399
-        if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
-        command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/metricbeat
-          source .buildkite/scripts/initCloudEnv.sh
-          echo "~~~ Running tests"
-          cd x-pack/metricbeat
-          mage build test
-        env:
-          ASDF_TERRAFORM_VERSION: 1.0.2
-          AWS_REGION: "eu-central-1"
-          MODULE_DIR: x-pack/metricbeat/module/aws
-          REPO: beats
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Cloud AWS (MODULE) Tests"
-
-  - wait: ~
-    # with PRs, we want to run packaging only if mandatory tests succeed
-    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
-    # this allows building DRA artifacts even if there is flakiness in mandatory tests
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - step: "x-pack-metricbeat-mandatory-tests"
-
-  - group: "x-pack/metricbeat Packaging"
-    key: "x-pack-metricbeat-packaging"
-    steps:
-      - label: ":ubuntu: x-pack/metricbeat Packaging Linux"
-        key: "packaging-linux"
-        command: |
-          cd x-pack/metricbeat
-          mage package
-        retry:
-          automatic:
-            - limit: 3
-        timeout_in_minutes: 20
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Packaging Linux"
-
-      - label: ":ubuntu: x-pack/metricbeat Packaging arm64"
-        key: "packaging-arm"
-        command: |
-          cd x-pack/metricbeat
-          mage package
-        retry:
-          automatic:
-            - limit: 3
-        timeout_in_minutes: 20
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        env:
-          PLATFORMS: "linux/arm64"
-          PACKAGES: "docker"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Packaging Linux arm64"
+#      - label: ":ubuntu: x-pack/metricbeat Python (MODULE) Integration Tests"
+#        key: "mandatory-python-int-test"
+#        command: |
+#          set -euo pipefail
+#          # defines the MODULE env var based on what's changed in a PR
+#          source .buildkite/scripts/changesets.sh
+#          defineModuleFromTheChangeSet x-pack/metricbeat
+#          echo "~~~ Running tests"
+#          cd x-pack/metricbeat && mage pythonIntegTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "x-pack/metricbeat/build/*.xml"
+#          - "x-pack/metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Python (MODULE) Integration Tests"
+#
+#      - label: ":windows: x-pack/metricbeat Win-2019 Unit Tests"
+#        command: |
+#          Set-Location -Path x-pack/metricbeat
+#          mage build unitTest
+#        key: "extended-win-2019-unit-tests"
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2019}"
+#          machineType: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "x-pack/metricbeat/build/*.xml"
+#          - "x-pack/metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Win-2019 Unit Tests"
+#
+#  - group: "x-pack/metricbeat Extended Windows Tests"
+#    key: "x-pack-metricbeat-extended-win-tests"
+#    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+#
+#    steps:
+#      - label: ":windows: x-pack/metricbeat Win-2016 Unit Tests"
+#        command: |
+#          Set-Location -Path x-pack/metricbeat
+#          mage build unitTest
+#        key: "mandatory-win-2016-unit-tests"
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2016}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "x-pack/metricbeat/build/*.xml"
+#          - "x-pack/metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Win-2016 Unit Tests"
+#
+#      - label: ":windows: x-pack/metricbeat Win 10 Unit Tests"
+#        command: |
+#          Set-Location -Path x-pack/metricbeat
+#          mage build unitTest
+#        key: "extended-win-10-unit-tests"
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_10}"
+#          machineType: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "x-pack/metricbeat/build/*.xml"
+#          - "x-pack/metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Win 10 Unit Tests"
+#
+#  - group: "x-pack/metricbeat Extended Tests"
+#    key: "x-pack-metricbeat-extended-tests"
+#    if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/
+#
+#    steps:
+#      - label: ":mac: x-pack/metricbeat macOS x86_64 Unit Tests"
+#        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#        command: |
+#          set -euo pipefail
+#          source .buildkite/scripts/install_macos_tools.sh
+#          cd x-pack/metricbeat && mage build unitTest
+#        retry:
+#          automatic:
+#            - limit: 3
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_X86_64}"
+#        artifact_paths:
+#          - "x-pack/metricbeat/build/*.xml"
+#          - "x-pack/metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: macOS x86_64 Unit Tests"
+#
+#      - label: ":ubuntu: x-pack/metricbeat Cloud (MODULE) Tests"
+#        key: "x-pack-metricbeat-extended-cloud-test"
+#        if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
+#        command: |
+#          set -euo pipefail
+#          # defines the MODULE env var based on what's changed in a PR
+#          source .buildkite/scripts/changesets.sh
+#          defineModuleFromTheChangeSet x-pack/metricbeat
+#          source .buildkite/scripts/initCloudEnv.sh
+#          echo "~~~ Running tests"
+#          cd x-pack/metricbeat
+#          mage build test
+#        env:
+#          ASDF_TERRAFORM_VERSION: 1.0.2
+#          MODULE_DIR: "x-pack/metricbeat/module/aws"
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+#        artifact_paths:
+#          - "x-pack/metricbeat/build/*.xml"
+#          - "x-pack/metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Cloud (MODULE) Tests"
+#
+#      - label: ":ubuntu: x-pack/metricbeat Cloud AWS (MODULE) Tests"
+#        key: "x-pack-metricbeat-extended-cloud-test-aws"
+##        skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3399"
+#        # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3399
+#        if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
+#        command: |
+#          set -euo pipefail
+#          # defines the MODULE env var based on what's changed in a PR
+#          source .buildkite/scripts/changesets.sh
+#          defineModuleFromTheChangeSet x-pack/metricbeat
+#          source .buildkite/scripts/initCloudEnv.sh
+#          echo "~~~ Running tests"
+#          cd x-pack/metricbeat
+#          mage build test
+#        env:
+#          ASDF_TERRAFORM_VERSION: 1.0.2
+#          AWS_REGION: "eu-central-1"
+#          MODULE_DIR: x-pack/metricbeat/module/aws
+#          REPO: beats
+#        agents:
+#          provider: "aws"
+#          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+#          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+#        artifact_paths:
+#          - "x-pack/metricbeat/build/*.xml"
+#          - "x-pack/metricbeat/build/*.json"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Cloud AWS (MODULE) Tests"
+#
+#  - wait: ~
+#    # with PRs, we want to run packaging only if mandatory tests succeed
+#    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+#    # this allows building DRA artifacts even if there is flakiness in mandatory tests
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+#    depends_on:
+#      - step: "x-pack-metricbeat-mandatory-tests"
+#
+#  - group: "x-pack/metricbeat Packaging"
+#    key: "x-pack-metricbeat-packaging"
+#    steps:
+#      - label: ":ubuntu: x-pack/metricbeat Packaging Linux"
+#        key: "packaging-linux"
+#        command: |
+#          cd x-pack/metricbeat
+#          mage package
+#        retry:
+#          automatic:
+#            - limit: 3
+#        timeout_in_minutes: 20
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        env:
+#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Packaging Linux"
+#
+#      - label: ":ubuntu: x-pack/metricbeat Packaging arm64"
+#        key: "packaging-arm"
+#        command: |
+#          cd x-pack/metricbeat
+#          mage package
+#        retry:
+#          automatic:
+#            - limit: 3
+#        timeout_in_minutes: 20
+#        agents:
+#          provider: "aws"
+#          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+#          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+#        env:
+#          PLATFORMS: "linux/arm64"
+#          PACKAGES: "docker"
+#        notify:
+#          - github_commit_status:
+#              context: "x-pack/metricbeat: Packaging Linux arm64"