[7.17](backport #38135) Migrate xpack packetbeat

.buildkite/hooks/pre-command

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
+PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 
 retry() {
   local retries=$1
@@ -33,7 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"
@@ -47,3 +48,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP
   BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
   export BEATS_AWS_ACCESS_KEY
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then
+    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET
+  fi
+fi

.buildkite/scripts/common.sh

@@ -22,6 +22,8 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 [ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")"
 [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
 [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
+[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")"
+[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"
 
 metricbeat_changeset=(
   "^metricbeat/.*"
@@ -105,6 +107,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in
   "beats-xpack-metricbeat")
     BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
     ;;
+  "beats-xpack-packetbeat")
+    BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]}
+    ;;
   *)
   echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet."
   ;;

.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh

@@ -31,8 +31,6 @@ steps:
           provider: "gcp"
           image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":python: Python Integration Tests"
@@ -42,8 +40,6 @@ steps:
           provider: "gcp"
           image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":windows: Windows Unit Tests - {{matrix.image}}"
@@ -106,8 +102,7 @@ else
 fi
 
 #TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved
-if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then
-# if [[ are_conditions_met_macos_tests ]]; then
+if are_conditions_met_aws_tests || are_conditions_met_macos_tests ; then
   cat >> $pipelineName <<- YAML
 
   - group: "Extended Tests"
@@ -140,8 +135,6 @@ if  are_conditions_met_aws_tests; then
           provider: "gcp"
           image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
 YAML
@@ -166,6 +159,8 @@ if are_conditions_met_packaging; then
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
         env:
           PLATFORMS: "${PACKAGING_PLATFORMS}"
 

.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh

@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.xpack-packetbeat-dynamic.yml"
+
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        key: "mandatory-linux-unit-test"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":linux: Ubuntu System Tests"
+        key: "mandatory-linux-system-test"
+        command: "cd $BEATS_PROJECT_NAME && mage systemTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":rhel: RHEL-9 Unit Tests"
+        key: "mandatory-rhel9-unit-test"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+
+      - label: ":windows: Windows Unit Tests - {{matrix.image}}"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "mandatory-win-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      ##  TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved
+      # - label: ":windows: Windows 2022 System Tests"
+      #   key: "mandatory-win-2022-system-tests"
+      #   command: ".buildkite/scripts/win_unit_tests.ps1 systemtest"
+      #   agents:
+      #     provider: "gcp"
+      #     image: "${IMAGE_WIN_2022}"
+      #     machineType: "${GCP_WIN_MACHINE_TYPE}"
+      #     disk_size: 100
+      #     disk_type: "pd-ssd"
+      #   artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h
+
+  - group: "Extended Windows Tests"
+    key: "extended-win-tests"
+    steps:
+
+      - label: ":windows: Windows Unit Tests - {{matrix.image}}"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "extended-win-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_10}"
+              - "${IMAGE_WIN_11}"
+              - "${IMAGE_WIN_2019}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      ##  TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved
+      # - label: ":windows: Windows 10 System Tests"
+      #   key: "extended-win-10-system-tests"
+      #   command: ".buildkite/scripts/win_unit_tests.ps1 systemtest"
+      #   agents:
+      #     provider: "gcp"
+      #     image: "${IMAGE_WIN_10}"
+      #     machineType: "${GCP_WIN_MACHINE_TYPE}"
+      #     disk_size: 100
+      #     disk_type: "pd-ssd"
+      #   artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
+fi
+
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+
+YAML
+fi
+
+if  are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "extended-macos-unit-tests"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if  are_conditions_met_arm_tests; then
+  cat >> $pipelineName <<- YAML
+      - label: ":linux: ARM Ubuntu Unit Tests"
+        key: "extended-arm64-unit-test"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+echo "Check and add the Packaging into the pipeline"
+if are_conditions_met_packaging; then
+  cat >> $pipelineName <<- YAML
+
+  - wait: ~
+    depends_on:
+      - step: "mandatory-tests"
+        allow_failure: false
+
+  - group: "Packaging"    # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
+    key: "packaging"
+    steps:
+      - label: ":linux: Packaging Linux"
+        key: "packaging-linux"
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        env:
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
+
+      - label: ":linux: Packaging ARM"
+        key: "packaging-arm"
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        env:
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
+          PACKAGES: "docker"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName

.buildkite/scripts/win_unit_tests.ps1

@@ -1,3 +1,7 @@
+param(
+    [string]$testType = "unittest"
+)
+
 $ErrorActionPreference = "Stop" # set -e
 $WorkFolder = $env:BEATS_PROJECT_NAME
 $WORKSPACE = Get-Location
@@ -120,6 +124,23 @@ function withNmap($version) {
     }
     Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait
 }
+function google_cloud_auth {
+    $tempFileName = "google-cloud-credentials.json"
+    $secretFileLocation = Join-Path $env:TEMP $tempFileName
+    $null = New-Item -ItemType File -Path $secretFileLocation
+    Set-Content -Path $secretFileLocation -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET
+    gcloud auth activate-service-account --key-file $secretFileLocation > $null 2>&1
+    $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation
+}
+
+function google_cloud_auth_cleanup {
+    if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) {
+        Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force
+        Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS
+    } else {
+        Write-Host "No GCP credentials were added"
+    }
+}
 
 fixCRLF
 
@@ -142,10 +163,23 @@ $env:MAGEFILE_CACHE = $magefile
 
 New-Item -ItemType Directory -Force -Path "build"
 
-if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") {
-    mage -w reader/etw build goUnitTest
-} else {
-    mage build unitTest
+if ($testType -eq "unittest") {
+    if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") {
+        mage -w reader/etw build goUnitTest
+    } else {
+        mage build unitTest
+    }
+}
+elseif ($testType -eq "systemtest") {
+    try {
+        google_cloud_auth
+        mage systemTest
+    } finally {
+        google_cloud_auth_cleanup
+    }
+}
+else {
+    Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'."
 }
 
 $EXITCODE=$LASTEXITCODE