From 9967eb9734e2b645458470d6045e074a8c3731b0 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 9 May 2024 13:47:40 +0300 Subject: [PATCH] Add MODULE definition for x-pack/dockerlogbeat Go Integration tests (#39235) (#39459) * added MODULE env var definition * removed MODULE setup from common.sh * updated pipeline and scripts * pr fixes * removed pipeline masking where irrelevant * updated packaging step & gh notif names (cherry picked from commit b6b428018cffbe6aee5db266ee8bda85d0edd9a9) Co-authored-by: Olga Naydyonock --- .buildkite/pipeline.yml | 2 +- .buildkite/scripts/common.sh | 7 +-- .buildkite/scripts/setenv.sh | 2 +- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 48 +++++++++++++------ 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2c00f376cf2..66fca45bbff 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -139,7 +139,7 @@ steps: commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" - - label: "Trigger Xpack/Dockerlogbeat" + - label: "Trigger x-pack/dockerlogbeat" if: build.pull_request.id != null plugins: - monorepo-diff#v1.0.1: diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b37ac7494fd..ed855df9970 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -24,11 +24,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" - -xpack_dockerlogbeat_changeset=( - "^x-pack/dockerlogbeat/.*" - ) - ci_changeset=( "^.buildkite/.*" ) @@ -478,7 +473,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index f806b131a7a..383ff6ff56b 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -49,7 +49,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then source .buildkite/scripts/common.sh # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index a64f7851913..f874f0c45e9 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -2,16 +2,14 @@ name: "beats-xpack-dockerlogbeat" env: - ASDF_MAGE_VERSION: 1.15.0 - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" @@ -21,13 +19,18 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + # Other deps + ASDF_MAGE_VERSION: 1.15.0 + steps: - - group: "Xpack/Dockerlogbeat Mandatory Tests" + - group: "x-pack/dockerlogbeat Mandatory Tests" key: "xpack-dockerlogbeat-mandatory-tests" steps: - - label: ":ubuntu: Xpack/Dockerlogbeat Ubuntu Unit Tests" + - label: ":ubuntu: x-pack/dockerlogbeat Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd x-pack/dockerlogbeat && mage build unitTest" + command: | + cd x-pack/dockerlogbeat + mage build unitTest retry: automatic: - limit: 3 @@ -42,9 +45,16 @@ steps: - github_commit_status: context: "x-pack/dockerlogbeat: Ubuntu Unit Tests" - - label: ":ubuntu: Xpack/Dockerlogbeat Go Integration Tests" + - label: ":ubuntu: x-pack/dockerlogbeat Go (Module) Integration Tests" key: "mandatory-int-test" - command: "cd x-pack/dockerlogbeat && mage goIntegTest" + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet x-pack/dockerlogbeat + echo "~~~ Running tests with env var MODULE=$$MODULE" + cd x-pack/dockerlogbeat + mage goIntegTest env: MODULE: $MODULE retry: @@ -59,7 +69,7 @@ steps: - "x-pack/dockerlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/dockerlogbeat: Go Integration Tests" + context: "x-pack/dockerlogbeat: Go (Module) Integration Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -69,29 +79,39 @@ steps: depends_on: - "xpack-dockerlogbeat-mandatory-tests" - - group: "Xpack/Dockerlogbeat Packaging" + - group: "x-pack/dockerlogbeat Packaging" key: "xpack-dockerlogbeat-packaging" steps: - - label: ":ubuntu: Xpack/Dockerlogbeat Packaging Linux X86" + - label: ":ubuntu: x-pack/dockerlogbeat Packaging Linux" key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true - command: "cd x-pack/dockerlogbeat && mage package" + command: | + cd x-pack/dockerlogbeat + mage package agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "x-pack/dockerlogbeat: Packaging Linux" - - label: ":linux: Xpack/Dockerlogbeat Packaging Linux ARM" + - label: ":linux: x-pack/dockerlogbeat Packaging Linux arm64" key: "auditbeat-package-linux-arm" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: true - command: "cd x-pack/dockerlogbeat && mage package" + command: | + cd x-pack/dockerlogbeat + mage package agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" + notify: + - github_commit_status: + context: "x-pack/dockerlogbeat: Packaging Linux arm64"