Merge main into feature/health-report-api branch

.buildkite/benchmark_marathon_pipeline.yml

@@ -0,0 +1,11 @@
+agents:
+  provider: gcp
+  imageProject: elastic-images-prod
+  image: family/platform-ingest-logstash-ubuntu-2204
+  machineType: "n2-standard-16"
+  diskSizeGb: 100
+  diskType: pd-ssd
+
+steps:
+  - label: "Benchmark Marathon"
+    command: .buildkite/scripts/benchmark/marathon.sh

.buildkite/health_report_tests_pipeline.yml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+agents:
+  image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
+  cpu: "2"
+  memory: "4Gi"
+  ephemeralStorage: "64Gi"
+
+steps:
+  # todo: add a step for 8.latest when Health Report MVP is landed
+  - label: ":logstash: Health Report Integration tests on main branch."
+    command: |
+      source .buildkite/scripts/common/vm-agent.sh
+      ./.buildkite/scripts/health-report-tests/main.sh
+    retry:
+      automatic:
+        - limit: 3

.buildkite/linux_jdk_matrix_pipeline.yml

@@ -60,20 +60,14 @@ steps:
             value: "adoptiumjdk_21"
           - label: "Adoptium JDK 17 (Eclipse Temurin)"
             value: "adoptiumjdk_17"
-          - label: "Adoptium JDK 11 (Eclipse Temurin)"
-            value: "adoptiumjdk_11"
           - label: "OpenJDK 21"
             value: "openjdk_21"
           - label: "OpenJDK 17"
             value: "openjdk_17"
-          - label: "OpenJDK 11"
-            value: "openjdk_11"
           - label: "Zulu 21"
             value: "zulu_21"
           - label: "Zulu 17"
             value: "zulu_17"
-          - label: "Zulu 11"
-            value: "zulu_11"
 
   - wait: ~
     if: build.source != "schedule" && build.source != "trigger_job"

.buildkite/pull-requests.json

@@ -5,7 +5,7 @@
         "pipeline_slug": "logstash-pull-request-pipeline",
         "allow_org_users": true,
         "allowed_repo_permissions": ["admin", "write"],
-        "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"],
+        "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"],
         "set_commit_status": true,
         "build_on_commit": true,
         "build_on_comment": true,

.buildkite/scripts/benchmark/README.md

@@ -0,0 +1,21 @@
+## Steps to set up GCP instance to run benchmark script
+- Create an instance "n2-standard-16" with Ubuntu image
+- Install docker
+  - `sudo snap install docker`
+  - `sudo usermod -a -G docker $USER`
+- Install jq
+- Install vault
+  - `sudo snap install vault`
+  - `vault login --method github`
+  - `vault kv get -format json secret/ci/elastic-logstash/benchmark`
+- Setup Elasticsearch index mapping and alias with `setup/*`
+- Import Kibana dashboard with `save-objects/*`
+- Run the benchmark script
+  - Send data to your own Elasticsearch. Customise `VAULT_PATH="secret/ci/elastic-logstash/your/path"`
+  - Run the script `main.sh` 
+    - or run in background `nohup bash -x main.sh > log.log 2>&1 &`
+
+## Notes
+- Benchmarks should only be compared using the same hardware setup.
+- Please do not send the test metrics to the benchmark cluster. You can set `VAULT_PATH` to send data and metrics to your own server.
+- Run `all.sh` as calibration which gives you a baseline of performance in different versions.

.buildkite/scripts/benchmark/config/uuid

@@ -0,0 +1 @@
+f74f1a28-25e9-494f-ba41-ca9f13d4446d