From 2cddce9121f6982edeba35738f5bb8a52cc8d197 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Fri, 10 May 2024 17:11:09 +0300 Subject: [PATCH] test module definition --- .buildkite/env-scripts/env.sh | 54 -- .buildkite/env-scripts/util.sh | 37 -- .../scripts/packaging/package-util.sh | 30 - .../filebeat/scripts/packaging/package.sh | 49 -- .../scripts/packaging/packaging-env.sh | 33 - .buildkite/heartbeat/heartbeat-pipeline.yml | 570 +++++++++--------- .buildkite/hooks/pre-command | 49 -- .buildkite/metricbeat/pipeline.yml | 546 ++++++++--------- .buildkite/scripts/changesets.sh | 4 +- .buildkite/scripts/define_module.sh | 65 +- .buildkite/scripts/moduleDefinition/go.mod | 3 + .buildkite/scripts/moduleDefinition/main.go | 102 ++++ metricbeat/module/linux/deleteme.asciidoc | 0 metricbeat/module/linux/deleteme.go | 0 metricbeat/module/linux/deleteme.png | 0 .../module/linux/deleteme1/deletemeagain.go | 0 .../logstash/logstash_integration_test.go | 1 + .../module/deleteme/subdir/deletemeagain.go | 1 + x-pack/auditbeat/module/system/deleteme.go | 1 + x-pack/metricbeat/module/deleteme.go | 1 + x-pack/metricbeat/module/deleteme/deletme.go | 1 + .../module/deleteme/subdir/deletemeagain.go | 1 + 22 files changed, 713 insertions(+), 835 deletions(-) delete mode 100644 .buildkite/env-scripts/env.sh delete mode 100755 .buildkite/filebeat/scripts/packaging/package-util.sh delete mode 100755 .buildkite/filebeat/scripts/packaging/package.sh delete mode 100755 .buildkite/filebeat/scripts/packaging/packaging-env.sh create mode 100644 .buildkite/scripts/moduleDefinition/go.mod create mode 100755 .buildkite/scripts/moduleDefinition/main.go create mode 100644 metricbeat/module/linux/deleteme.asciidoc create mode 100644 metricbeat/module/linux/deleteme.go create mode 100644 metricbeat/module/linux/deleteme.png create mode 100644 metricbeat/module/linux/deleteme1/deletemeagain.go create mode 100644 x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go create mode 100644 x-pack/auditbeat/module/system/deleteme.go create mode 100644 x-pack/metricbeat/module/deleteme.go create mode 100644 x-pack/metricbeat/module/deleteme/deletme.go create mode 100644 x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh deleted file mode 100644 index dca5248f744..00000000000 --- a/.buildkite/env-scripts/env.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/env-scripts/util.sh - -DOCS_CHANGESET="^.*\.(asciidoc|md)$ -deploy/kubernetes/.*-kubernetes.yaml" -PACKAGING_CHANGESET="^dev-tools/packaging/ -^.go-version" - -REPO="beats" -WORKSPACE="$(pwd)" -BIN="${WORKSPACE}/bin" -HW_TYPE="$(uname -m)" -PLATFORM_TYPE="$(uname)" -SNAPSHOT="true" -PYTEST_ADDOPTS="" - -SETUP_GVM_VERSION="v0.5.1" -ASDF_MAGE_VERSION="1.14.0" -SETUP_WIN_PYTHON_VERSION="3.11.0" - -# Docker & DockerHub -DOCKER_COMPOSE_VERSION="1.21.0" -DOCKER_REGISTRY="docker.elastic.co" - -ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET") -PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET") -GO_MOD_CHANGES=$(changeset_applies "^go.mod") - -KIND_VERSION="v0.20.0" -KUBECONFIG="${WORKSPACE}/kubecfg" - -export WORKSPACE -export BIN -export HW_TYPE -export PLATFORM_TYPE -export SNAPSHOT -export PYTEST_ADDOPTS - -export SETUP_GVM_VERSION -export ASDF_MAGE_VERSION -export SETUP_WIN_PYTHON_VERSION - -export DOCKER_COMPOSE_VERSION -export DOCKER_REGISTRY - -export ONLY_DOCS -export PACKAGING_CHANGES -export GO_MOD_CHANGES - -export KIND_VERSION -export KUBECONFIG - -add_bin_path diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 261f2bbee2d..68fd08a75df 100755 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -102,40 +102,3 @@ set_git_config() { git config user.name "${GITHUB_USERNAME_SECRET}" git config user.email "${GITHUB_EMAIL_SECRET}" } - -changeset_applies() { - local changeset=$1 - if are_files_changed "$changeset"; then - echo true - else - echo false - fi -} - -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then - unset "$var" - fi - done -} - -google_cloud_logout_active_account() { - local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null) - if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then - echo "Logging out from GCP for active account" - gcloud auth revoke $active_account > /dev/null 2>&1 - else - echo "No active GCP accounts found." - fi - if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then - unset GOOGLE_APPLICATION_CREDENTIALS - cleanup - fi -} - -cleanup() { - echo "Deleting temporary files..." - rm -rf ${BIN}/${TMP_FOLDER}.* - echo "Done." -} diff --git a/.buildkite/filebeat/scripts/packaging/package-util.sh b/.buildkite/filebeat/scripts/packaging/package-util.sh deleted file mode 100755 index cb409e8d484..00000000000 --- a/.buildkite/filebeat/scripts/packaging/package-util.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -is_pr() { - if [[ $BUILDKITE_PULL_REQUEST != false ]]; then - return 0 - else - return 1 - fi -} - -define_tags() { - aliasVersion="${VERSION%.*}${IMG_POSTFIX}" - tags=("${BUILDKITE_COMMIT}") - - if is_pr; then - tags+=("pr-${GITHUB_PR_NUMBER}") - else - tags+=("${SOURCE_TAG}" "${aliasVersion}") - fi -} - -check_is_arm() { - if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then - is_arm="-arm" - else - is_arm="" - fi -} diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh deleted file mode 100755 index f338e97c0b4..00000000000 --- a/.buildkite/filebeat/scripts/packaging/package.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh -source .buildkite/filebeat/scripts/packaging/package-util.sh - -IMG_POSTFIX="-SNAPSHOT" -VARIANTS=("" "-ubi" "-oss") -VERSION="$(make get-version)" -SOURCE_TAG+="${VERSION}${IMG_POSTFIX}" -BEAT_NAME="filebeat" -TARGET="observability-ci/${BEAT_NAME}" - -echo "--- Creating package" -mage -d filebeat package - -#echo "--- Distribution list" -#dir="filebeat/build/distributions" -#buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512" -# -#echo "--- Docker image list" -#docker images -# -#define_tags -#check_is_arm -# -#echo "--- Tag & Push" -#for variant in "${VARIANTS[@]}"; do -# source="beats/${BEAT_NAME}${variant}" -# -# for tag in "${tags[@]}"; do -# targetTag=$tag${is_arm} -# -# sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}" -# targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}" -# # Remove following lines once beats fully migrated -# targetName="${targetName}-buildkite" -# -# if docker image inspect "${sourceName}" &>/dev/null; then -# echo "Source name: $sourceName Target name: $targetName" -# docker tag "$sourceName" "$targetName" -## docker push "$targetName" -# -# else -# echo "Docker image ${sourceName} does not exist" -# fi -#done -#done diff --git a/.buildkite/filebeat/scripts/packaging/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh deleted file mode 100755 index 02c942361b4..00000000000 --- a/.buildkite/filebeat/scripts/packaging/packaging-env.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/env-scripts/util.sh - -docs_changeset="^.*\.(asciidoc|md)$ -deploy/kubernetes/.*-kubernetes.yaml" -packaging_changeset="^dev-tools/packaging/ -^.go-version" - -VERSION=$(make get-version | tr -d '\n') -ONLY_DOCS=$(changeset_applies "$docs_changeset") -PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset") -GO_MOD_CHANGES=$(changeset_applies "^go.mod") -# Change the postfix to -SNAPSHOT, once Jenkins is disabled - - -export PACKAGING_CHANGES -export ONLY_DOCS -export GO_MOD_CHANGES -export DOCKER_REGISTRY -export VERSION -export REPO -#export IMG_POSTFIX - - -#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH - -#IMAGES -> foreach = tag and push -# registry: ${REGISTRY}, -# sourceTag: calculate_tags->sourceTag, -# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS -# source: ${SOURCE}, -# target: ${TARGET} diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 7fe5cce11c1..fb5f881d878 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -30,81 +30,81 @@ steps: key: "heartbeat-mandatory-tests" steps: - - label: ":ubuntu: Heartbeat Unit Tests" - command: | - cd heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Ubuntu Unit Tests" - - - label: ":rhel: Heartbeat Rhel9 Unit Tests" - command: | - cd heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Rhel9 Unit Tests" - - - label: ":windows: Win-2016 Unit Tests" - key: "windows-2016" - command: | - Set-Location -Path heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_type: "pd-ssd" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Win-2016 Unit Tests" - - - label: ":windows: Heartbeat Win-2022 Unit Test" - key: "windows-2022" - command: | - Set-Location -Path heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_type: "pd-ssd" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Win-2022 Unit Tests" +# - label: ":ubuntu: Heartbeat Unit Tests" +# command: | +# cd heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_DEFAULT_MACHINE_TYPE}" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Ubuntu Unit Tests" +# +# - label: ":rhel: Heartbeat Rhel9 Unit Tests" +# command: | +# cd heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_RHEL9}" +# machineType: "${GCP_DEFAULT_MACHINE_TYPE}" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Rhel9 Unit Tests" +# +# - label: ":windows: Win-2016 Unit Tests" +# key: "windows-2016" +# command: | +# Set-Location -Path heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2016}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_type: "pd-ssd" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Win-2016 Unit Tests" +# +# - label: ":windows: Heartbeat Win-2022 Unit Test" +# key: "windows-2022" +# command: | +# Set-Location -Path heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2022}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_type: "pd-ssd" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Win-2022 Unit Tests" - label: ":ubuntu: Heartbeat Go (Module) Integration Tests" command: | @@ -116,6 +116,10 @@ steps: source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet heartbeat + source .buildkite/scripts/define_module.sh + defineModule heartbeat + # echo "~~~ New MODULE definition is $$NEW_MODULE" + echo "~~~ Running tests" cd heartbeat mage goIntegTest @@ -133,213 +137,211 @@ steps: - github_commit_status: context: "heartbeat: Go Integration Tests" - - label: ":ubuntu: Heartbeat Python (Module) Integration Tests" - command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet heartbeat - - echo "~~~ Running tests" - cd heartbeat - mage pythonIntegTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Python Integration Tests" - - - group: "Heartbeat Extended Tests" - key: "heartbeat-extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ - - steps: - - label: ":linux: Heartbeat arm64 Unit Tests" - key: "arm-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: | - cd heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "heartbeat/build/*.xml" - notify: - - github_commit_status: - context: "heartbeat: Linux arm64 Unit Tests" - - - group: "Heartbeat Extended Testing MacOS" - key: "heartbeat-extended-tests-macos" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ +# - label: ":ubuntu: Heartbeat Python (Module) Integration Tests" +# command: | +# set -euo pipefail +# # defines the MODULE env var based on what's changed in a PR +# source .buildkite/scripts/changesets.sh +# defineModuleFromTheChangeSet heartbeat +# +# source .buildkite/scripts/define_module.sh +# defineModule heartbeat +# # echo "~~~ New MODULE definition is $$NEW_MODULE" +# +# echo "~~~ Running tests" +# cd heartbeat +# mage pythonIntegTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Python Integration Tests" - steps: - - label: ":mac: Heartbeat MacOS Unit Tests" - - label: ":mac: macOS x86_64 Unit Tests" - key: "macos-extended" - if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: macOS x86_64 Unit Tests" - - - label: ":mac: macOS arm64 Unit Tests" - key: "macos-arm-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: macOS arm64 Unit Tests" - - - group: "Heartbeat Windows Extended Testing" - key: "heartbeat-extended-tests-win" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - - steps: - - label: ":windows: Win-2019 Unit Tests" - key: "heartbeat-win-extended-2019" - command: | - Set-Location -Path heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_type: "pd-ssd" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Win-2019 Unit Tests" - - - label: ":windows: Win-11 Unit Tests" - key: "heartbeat-windows-extended-11" - command: | - Set-Location -Path heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_type: "pd-ssd" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Win-11 Unit Tests" - - - label: ":windows: Win-10 Unit Tests" - key: "heartbeat-windows-extended-10" - command: | - Set-Location -Path heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_type: "pd-ssd" - artifact_paths: - - "heartbeat/build/*.xml" - - "heartbeat/build/*.json" - notify: - - github_commit_status: - context: "heartbeat: Win-10 Unit Tests" - - - wait: ~ - # with PRs, we want to run packaging only if mandatory tests succeed - # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in mandatory tests - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: "heartbeat-mandatory-tests" - - - group: "Heartbeat Packaging" - key: "heartbeat-packaging" - steps: - - label: ":ubuntu: Heartbeat Packaging Linux" - key: "heartbeat-package-linux" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - cd heartbeat - mage package - retry: - automatic: - - limit: 3 - timeout_in_minutes: 20 - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - notify: - - github_commit_status: - context: "heartbeat: Packaging Linux" - - - label: ":linux: Heartbeat Packaging Linux arm64" - key: "heartbeat-package-linux-arm" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: true - command: | - cd heartbeat - mage package - retry: - automatic: - - limit: 3 - timeout_in_minutes: 20 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - notify: - - github_commit_status: - context: "heartbeat: Packaging Linux arm64" +# - group: "Heartbeat Extended Tests" +# key: "heartbeat-extended-tests" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ +# +# steps: +# - label: ":linux: Heartbeat arm64 Unit Tests" +# key: "arm-extended" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ +# command: | +# cd heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "aws" +# imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# artifact_paths: "heartbeat/build/*.xml" +# notify: +# - github_commit_status: +# context: "heartbeat: Linux arm64 Unit Tests" +# +# - label: ":mac: macOS x86_64 Unit Tests" +# key: "macos-extended" +# if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ +# command: | +# set -euo pipefail +# source .buildkite/scripts/install_macos_tools.sh +# cd heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_X86_64}" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: macOS x86_64 Unit Tests" +# +# - label: ":mac: macOS arm64 Unit Tests" +# key: "macos-arm-extended" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ +# command: | +# set -euo pipefail +# source .buildkite/scripts/install_macos_tools.sh +# cd heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_ARM}" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: macOS arm64 Unit Tests" +# +# - group: "Heartbeat Windows Extended Testing" +# key: "heartbeat-extended-tests-win" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ +# +# steps: +# - label: ":windows: Win-2019 Unit Tests" +# key: "heartbeat-win-extended-2019" +# command: | +# Set-Location -Path heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2019}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_type: "pd-ssd" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Win-2019 Unit Tests" +# +# - label: ":windows: Win-11 Unit Tests" +# key: "heartbeat-windows-extended-11" +# command: | +# Set-Location -Path heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_11}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_type: "pd-ssd" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Win-11 Unit Tests" +# +# - label: ":windows: Win-10 Unit Tests" +# key: "heartbeat-windows-extended-10" +# command: | +# Set-Location -Path heartbeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_10}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_type: "pd-ssd" +# artifact_paths: +# - "heartbeat/build/*.xml" +# - "heartbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "heartbeat: Win-10 Unit Tests" +# +# - wait: ~ +# # with PRs, we want to run packaging only if mandatory tests succeed +# # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests +# # this allows building DRA artifacts even if there is flakiness in mandatory tests +# if: build.env("BUILDKITE_PULL_REQUEST") != "false" +# depends_on: "heartbeat-mandatory-tests" +# +# - group: "Heartbeat Packaging" +# key: "heartbeat-packaging" +# steps: +# - label: ":ubuntu: Heartbeat Packaging Linux" +# key: "heartbeat-package-linux" +# env: +# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +# SNAPSHOT: true +# command: | +# cd heartbeat +# mage package +# retry: +# automatic: +# - limit: 3 +# timeout_in_minutes: 20 +# agents: +# provider: gcp +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# notify: +# - github_commit_status: +# context: "heartbeat: Packaging Linux" +# +# - label: ":linux: Heartbeat Packaging Linux arm64" +# key: "heartbeat-package-linux-arm" +# env: +# PLATFORMS: "linux/arm64" +# PACKAGES: "docker" +# SNAPSHOT: true +# command: | +# cd heartbeat +# mage package +# retry: +# automatic: +# - limit: 3 +# timeout_in_minutes: 20 +# agents: +# provider: "aws" +# imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# notify: +# - github_commit_status: +# context: "heartbeat: Packaging Linux arm64" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 9a88715cded..428b7f84b53 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/util.sh - # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" @@ -11,53 +9,6 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then - source .buildkite/env-scripts/env.sh - if [[ -z "${GO_VERSION-""}" ]]; then - export GO_VERSION=$(cat "${WORKSPACE}/.go-version") - fi - - # Images with prefix "platform-ingest-beats-*" has Go pre-setup. - # Image itself takes care of Go version download/installation based on .go-version file - if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then - if [[ -z "${GO_VERSION-""}" ]]; then - GO_VERSION=$(cat "${WORKSPACE}/.go-version") - export GO_VERSION - fi - - # To prevent "OSError: [Errno 24] Too many open files" - ulimit -Sn 150000 - echo "--- Ulimit: $(ulimit)" - - echo "--- Setting up environment" - add_bin_path - with_go - with_mage - fi - - if [[ "$BUILDKITE_STEP_KEY" == *packaging* ]]; then - DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") - DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") - GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) - - export DOCKER_USERNAME_SECRET - export DOCKER_PASSWORD_SECRET - export GITHUB_TOKEN_SECRET - - docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null - - github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH}) - github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH}) - - git config user.name "$github_username" - git config user.email "$github_email" - fi - - if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then - ulimit -Sn 30000 - fi -fi - ENABLED_BEATS_PIPELINES_SLUGS=( "auditbeat" "heartbeat" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index ed9fb14f3d4..a91a2697646 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -33,75 +33,82 @@ steps: - group: "Metricbeat Mandatory Tests" key: "metricbeat-mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd metricbeat && mage build unitTest" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Ubuntu Unit Tests" +# - label: ":linux: Ubuntu Unit Tests" +# key: "mandatory-linux-unit-test" +# command: "cd metricbeat && mage build unitTest" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_DEFAULT_MACHINE_TYPE}" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Ubuntu Unit Tests" +# +# - label: ":go: Go Integration Tests" +# key: "mandatory-int-test" +# command: | +# set -euo pipefail +# # defines the MODULE env var based on what's changed in a PR +# source .buildkite/scripts/changesets.sh +# defineModuleFromTheChangeSet metricbeat +# echo "~~~ Will run tests with env var MODULE=$$MODULE" +# +# # TODO move this section to base image / pre-command hook +# echo "~~~ Installing kind" +# asdf plugin add kind +# asdf install kind $ASDF_KIND_VERSION +# +# .buildkite/deploy/kubernetes/scripts/kind-setup.sh +# +# echo "~~~ Running tests" +# export KUBECONFIG="$$PWD/kubecfg" +# cd metricbeat && mage goIntegTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Go Integration Tests" - - label: ":go: Go Integration Tests" - key: "mandatory-int-test" + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" + echo "~~~ OLD MODULE definition is $$MODULE" - # TODO move this section to base image / pre-command hook - echo "~~~ Installing kind" - asdf plugin add kind - asdf install kind $ASDF_KIND_VERSION + source .buildkite/scripts/define_module.sh + defineModule metricbeat + echo "~~~ New MODULE definition is $$MODULE" - .buildkite/deploy/kubernetes/scripts/kind-setup.sh + MODULE=$(go run .buildkite/scripts/moduleDefinition/main.go metricbeat) - echo "~~~ Running tests" - export KUBECONFIG="$$PWD/kubecfg" - cd metricbeat && mage goIntegTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Go Integration Tests" + export MODULE + echo "~~~ GO MODULE definition is $$MODULE" - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" - - # TODO move this section to base image / pre-command hook echo "~~~ Installing kind" asdf plugin add kind asdf install kind $ASDF_KIND_VERSION - .buildkite/deploy/kubernetes/scripts/kind-setup.sh - echo "~~~ Running tests" - export KUBECONFIG="$$PWD/kubecfg" - cd metricbeat && mage pythonIntegTest +# echo "~~~ Running tests" +# export KUBECONFIG="$$PWD/kubecfg" +# cd metricbeat && mage pythonIntegTest retry: automatic: - limit: 3 @@ -116,218 +123,219 @@ steps: - github_commit_status: context: "metricbeat: Python Integration Tests" - - label: ":negative_squared_cross_mark: Cross compile" - key: "mandatory-cross-compile" - command: "make -C metricbeat crosscompile" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Cross compile" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path metricbeat - mage build unitTest - key: "mandatory-win-2016-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path metricbeat - mage build unitTest - key: "mandatory-win-2022-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Windows 2022 Unit Tests" - - - group: "Metricbeat Extended Windows Tests" - key: "metricbeat-extended-win-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path metricbeat - mage build unitTest - key: "extended-win-10-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Extended Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path metricbeat - mage build unitTest - key: "extended-win-11-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Extended Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path metricbeat - mage build unitTest - key: "extended-win-2019-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Extended Windows 2019 Unit Tests" - - - group: "Metricbeat Extended MacOS Tests" - key: "metricbeat-extended-macos-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - steps: - - label: ":mac: MacOS x64_64 Unit Tests" - key: "extended-macos-x64-64-unit-tests" - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd metricbeat && mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Extended MacOS x86_64 Unit Tests" - - - label: ":mac: MacOS arm64 Unit Tests" - key: "extended-macos-arm64-unit-tests" - skip: "due to https://github.com/elastic/beats/issues/33035" - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd metricbeat && mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "metricbeat/build/*.xml" - - "metricbeat/build/*.json" - notify: - - github_commit_status: - context: "metricbeat: Extended MacOS arm64 Unit Tests" - - - wait: ~ - # with PRs, we want to run packaging only if mandatory tests succeed - # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in mandatory tests - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: "metricbeat-mandatory-tests" - - - group: "Metricbeat Packaging" - key: "metricbeat-packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd metricbeat && mage package" - retry: - automatic: - - limit: 3 - timeout_in_minutes: 20 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - notify: - - github_commit_status: - context: "metricbeat: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd metricbeat && mage package" - retry: - automatic: - - limit: 3 - timeout_in_minutes: 20 - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "metricbeat: Packaging Linux ARM" +# - label: ":negative_squared_cross_mark: Cross compile" +# key: "mandatory-cross-compile" +# command: "make -C metricbeat crosscompile" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_DEFAULT_MACHINE_TYPE}" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Cross compile" +# +# - label: ":windows: Windows 2016 Unit Tests" +# command: | +# Set-Location -Path metricbeat +# mage build unitTest +# key: "mandatory-win-2016-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2016}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Windows 2016 Unit Tests" +# +# - label: ":windows: Windows 2022 Unit Tests" +# command: | +# Set-Location -Path metricbeat +# mage build unitTest +# key: "mandatory-win-2022-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2022}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Windows 2022 Unit Tests" +# +# - group: "Metricbeat Extended Windows Tests" +# key: "metricbeat-extended-win-tests" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ +# steps: +# - label: ":windows: Windows 10 Unit Tests" +# command: | +# Set-Location -Path metricbeat +# mage build unitTest +# key: "extended-win-10-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_10}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Extended Windows 10 Unit Tests" +# +# - label: ":windows: Windows 11 Unit Tests" +# command: | +# Set-Location -Path metricbeat +# mage build unitTest +# key: "extended-win-11-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_11}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Extended Windows 11 Unit Tests" +# +# - label: ":windows: Windows 2019 Unit Tests" +# command: | +# Set-Location -Path metricbeat +# mage build unitTest +# key: "extended-win-2019-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2019}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Extended Windows 2019 Unit Tests" +# +# - group: "Metricbeat Extended MacOS Tests" +# key: "metricbeat-extended-macos-tests" +# if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ +# steps: +# - label: ":mac: MacOS x64_64 Unit Tests" +# key: "extended-macos-x64-64-unit-tests" +# command: | +# set -euo pipefail +# source .buildkite/scripts/install_macos_tools.sh +# cd metricbeat && mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_X86_64}" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Extended MacOS x86_64 Unit Tests" +# +# - label: ":mac: MacOS arm64 Unit Tests" +# key: "extended-macos-arm64-unit-tests" +# skip: "due to https://github.com/elastic/beats/issues/33035" +# command: | +# set -euo pipefail +# source .buildkite/scripts/install_macos_tools.sh +# cd metricbeat && mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_ARM}" +# artifact_paths: +# - "metricbeat/build/*.xml" +# - "metricbeat/build/*.json" +# notify: +# - github_commit_status: +# context: "metricbeat: Extended MacOS arm64 Unit Tests" +# +# +# - wait: ~ +# # with PRs, we want to run packaging only if mandatory tests succeed +# # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests +# # this allows building DRA artifacts even if there is flakiness in mandatory tests +# if: build.env("BUILDKITE_PULL_REQUEST") != "false" +# depends_on: "metricbeat-mandatory-tests" +# +# - group: "Metricbeat Packaging" +# key: "metricbeat-packaging" +# steps: +# - label: ":linux: Packaging Linux" +# key: "packaging-linux" +# command: "cd metricbeat && mage package" +# retry: +# automatic: +# - limit: 3 +# timeout_in_minutes: 20 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# env: +# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +# notify: +# - github_commit_status: +# context: "metricbeat: Packaging Linux" +# +# - label: ":linux: Packaging ARM" +# key: "packaging-arm" +# command: "cd metricbeat && mage package" +# retry: +# automatic: +# - limit: 3 +# timeout_in_minutes: 20 +# agents: +# provider: "aws" +# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# env: +# PLATFORMS: "linux/arm64" +# PACKAGES: "docker" +# notify: +# - github_commit_status: +# context: "metricbeat: Packaging Linux ARM" diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh index 5c6fd6c7b0d..de7cad8e710 100644 --- a/.buildkite/scripts/changesets.sh +++ b/.buildkite/scripts/changesets.sh @@ -69,9 +69,11 @@ defineModuleFromTheChangeSet() { done # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993 - if [[ ! -z $changed_modules ]]; then + if [[ ! -z $changed_modules ]]; then export MODULE="${changed_modules}" echo "~~~ Set env var MODULE to [$MODULE]" echo "~~~ Resuming commands" + else + export MODULE="null" fi } diff --git a/.buildkite/scripts/define_module.sh b/.buildkite/scripts/define_module.sh index d7187b58901..6134607f82b 100755 --- a/.buildkite/scripts/define_module.sh +++ b/.buildkite/scripts/define_module.sh @@ -4,52 +4,59 @@ set -euo pipefail OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -BEAT_PATH=$1 -MODULE='' definePattern() { - pattern="${XPACK_MODULE_PATTERN}" + pattern="${OSS_MODULE_PATTERN}" - if [[ "$BEAT_PATH" == *"x-pack/"* ]]; then - pattern="${OSS_MODULE_PATTERN}" + if [[ "$beatPath" == *"x-pack/"* ]]; then + pattern="${XPACK_MODULE_PATTERN}" fi } defineExclusions() { - local transformedDirectory=${BEAT_PATH//\//\\\/} - local exclusion="((?!^${transformedDirectory}\\/).)*\$" - exclude="^(${exclusion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" + exclude="^$beatPath\/module\/(.*(?/dev/null + beatPath=$1 + + definePattern + defineExclusions + defineFromCommit + getMatchingModules - export NEW_MODULE + if [ "${#modulesMatched[@]}" -gt 0 ]; then + addToModuleEnvVar + export MODULE=$modules + else + echo "~~~ No changes in modules for $beatPath" + fi } diff --git a/.buildkite/scripts/moduleDefinition/go.mod b/.buildkite/scripts/moduleDefinition/go.mod new file mode 100644 index 00000000000..5eaf5ca38b1 --- /dev/null +++ b/.buildkite/scripts/moduleDefinition/go.mod @@ -0,0 +1,3 @@ +module github.com/elastic/beats/module + +go 1.21.9 diff --git a/.buildkite/scripts/moduleDefinition/main.go b/.buildkite/scripts/moduleDefinition/main.go new file mode 100755 index 00000000000..0a5d25f39f5 --- /dev/null +++ b/.buildkite/scripts/moduleDefinition/main.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "regexp" + "strings" +) + +func main() { + if len(os.Args) < 2 { + fmt.Println("Argument required: beatPath") + os.Exit(1) + } + + beatPath := os.Args[1] + + var modulePattern string + if strings.Contains(beatPath, "x-pack") { + modulePattern = "^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" + } else { + modulePattern = "^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" + } + + moduleRegex, err := regexp.Compile(modulePattern) + if err != nil { + log.Fatal("failed to compile regex: " + err.Error()) + } + + modules := make(map[string]bool) + for _, line := range getDiff() { + if !isAsciiOrPng(line) { + matches := moduleRegex.FindStringSubmatch(line) + if len(matches) > 0 { + modules[matches[1]] = true + } + } + } + + keys := make([]string, len(modules)) + i := 0 + for k := range modules { + keys[i] = k + i++ + } + + fmt.Println(strings.Join(keys, ",")) +} + +func isAsciiOrPng(file string) bool { + return strings.HasSuffix(file, ".asciidoc") || strings.HasSuffix(file, ".png") +} + +func getDiff() []string { + cmd := exec.Command("git", "diff", "--name-only", getCommitRange()) + output, err := cmd.Output() + if err != nil { + fmt.Printf("Error: %s", err) + os.Exit(1) + } + + return strings.Split(string(output), "\n") +} + +func getFromCommit() string { + baseBranch := os.Getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH") + branch := os.Getenv("BUILDKITE_BRANCH") + commit := os.Getenv("BUILDKITE_COMMIT") + + if baseBranch != "" { + return fmt.Sprintf("origin/%s", baseBranch) + } + + if branch != "" { + return fmt.Sprintf("origin/%s", branch) + } + + previousCommit := getPreviousCommit() + if previousCommit != "" { + return previousCommit + } + + return commit +} + +func getPreviousCommit() string { + cmd := exec.Command("git", "rev-parse", "HEAD^") + output, err := cmd.Output() + if err != nil { + fmt.Println("Error: ", err) + os.Exit(1) + } + + return string(output) +} + +func getCommitRange() string { + commit := os.Getenv("BUILDKITE_COMMIT") + return fmt.Sprintf("%s...%s", getFromCommit(), commit) +} diff --git a/metricbeat/module/linux/deleteme.asciidoc b/metricbeat/module/linux/deleteme.asciidoc new file mode 100644 index 00000000000..e69de29bb2d diff --git a/metricbeat/module/linux/deleteme.go b/metricbeat/module/linux/deleteme.go new file mode 100644 index 00000000000..e69de29bb2d diff --git a/metricbeat/module/linux/deleteme.png b/metricbeat/module/linux/deleteme.png new file mode 100644 index 00000000000..e69de29bb2d diff --git a/metricbeat/module/linux/deleteme1/deletemeagain.go b/metricbeat/module/linux/deleteme1/deletemeagain.go new file mode 100644 index 00000000000..e69de29bb2d diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index 5865867cf1f..db430268b2d 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -63,6 +63,7 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0].BeatEvent("logstash", metricSet).Fields.StringToPrint()) + t.Logf("%s just to test changesets", f.Name()) }) } } diff --git a/x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go b/x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go new file mode 100644 index 00000000000..30bc49fa39e --- /dev/null +++ b/x-pack/auditbeat/module/deleteme/subdir/deletemeagain.go @@ -0,0 +1 @@ +package subdir diff --git a/x-pack/auditbeat/module/system/deleteme.go b/x-pack/auditbeat/module/system/deleteme.go new file mode 100644 index 00000000000..9b140a34dd3 --- /dev/null +++ b/x-pack/auditbeat/module/system/deleteme.go @@ -0,0 +1 @@ +package system diff --git a/x-pack/metricbeat/module/deleteme.go b/x-pack/metricbeat/module/deleteme.go new file mode 100644 index 00000000000..b0b78bfd7a2 --- /dev/null +++ b/x-pack/metricbeat/module/deleteme.go @@ -0,0 +1 @@ +package module diff --git a/x-pack/metricbeat/module/deleteme/deletme.go b/x-pack/metricbeat/module/deleteme/deletme.go new file mode 100644 index 00000000000..935f8ef80cd --- /dev/null +++ b/x-pack/metricbeat/module/deleteme/deletme.go @@ -0,0 +1 @@ +package deleteme diff --git a/x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go b/x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go new file mode 100644 index 00000000000..30bc49fa39e --- /dev/null +++ b/x-pack/metricbeat/module/deleteme/subdir/deletemeagain.go @@ -0,0 +1 @@ +package subdir