Merge branch 'main' into go-vet-1.24

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -6,7 +6,7 @@ env:
   MODULE: "kubernetes"
 
   # Other deps
-  ASDF_KIND_VERSION: "0.20.0"
+  ASDF_KIND_VERSION: "0.24.0"
 
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 

.buildkite/filebeat/filebeat-pipeline.yml

@@ -24,7 +24,7 @@ env:
   ASDF_MAGE_VERSION: 1.15.0
 
   # Integration Tests
-  K8S_VERSION: "v1.29.0"
+  K8S_VERSION: "v1.31.0"
   ASDF_KIND_VERSION: "0.20.0"
 
   # Unit tests

.buildkite/metricbeat/pipeline.yml

@@ -21,7 +21,7 @@ env:
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
   # Integration Tests
-  K8S_VERSION: "v1.29.0"
+  K8S_VERSION: "v1.31.0"
   ASDF_KIND_VERSION: "0.20.0"
 
   # Module Tests

.mergify.yml

@@ -355,6 +355,19 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.16 branch
+    conditions:
+      - merged
+      - label~=^(backport-v8.16.0|backport-8.16)$
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.16"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 8.x branch
     conditions:
       - merged

CHANGELOG-developer.next.asciidoc

@@ -70,6 +70,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Debug log entries from the acker (`stateful ack ...` or `stateless ack ...`) removed. {pull}39672[39672]
 - Rename x-pack/filebeat websocket input to streaming. {issue}40264[40264] {pull}40421[40421]
 - Journald input now calls `journalctl` instead of using `github.com/coreos/go-systemd/[email protected]/sdjournal`, the CGO dependency has been removed from Filebeat {pull}40061[40061]
+- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files, however the ingest pipeline sets it back to the original input (log or journald). {pull}41246[41246]
 
 ==== Bugfixes
 
@@ -208,6 +209,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] 
 - Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936]
 - Add field redaction package. {pull}40997[40997]
+- Add support for marked redaction to x-pack/filebeat/input/internal/private {pull}41212[41212]
+- Add support for collecting Okta role and factor data for users with filebeat entityanalytics input. {pull}41044[41044]
 
 ==== Deprecated
 

CHANGELOG.next.asciidoc

@@ -46,6 +46,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added `container.image.name` to `journald` Filebeat input's Docker-specific translated fields. {pull}40450[40450]
 - Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099]
 - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089]
+- The performance of ingesting SQS data with the S3 input has improved by up to 60x for queues with many small events. `max_number_of_messages` config for SQS mode is now ignored, as the new design no longer needs a manual cap on messages. Instead, use `number_of_workers` to scale ingestion rate in both S3 and SQS modes. The increased efficiency may increase network bandwidth consumption, which can be throttled by lowering `number_of_workers`. It may also increase number of events stored in memory, which can be throttled by lowering the configured size of the internal queue. {pull}40699[40699]
+- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files. {pull}41061[41061]
 
 
 *Heartbeat*
@@ -120,7 +122,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Auditbeat*
 
-
+- Request status from a separate socket to avoid data congestion {pull}41207[41207]
 
 *Filebeat*
 
@@ -324,6 +326,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Improved GCS input documentation. {pull}41143[41143]
 - Add CSV decoding capacity to azureblobstorage input {pull}40978[40978]
 - Add CSV decoding capacity to gcs input {pull}40979[40979]
+- Add support to source AWS cloudwatch logs from linked accounts. {pull}41188[41188]
+- Jounrald input now supports filtering by facilities {pull}41061[41061]
+- System module now supports reading from jounrald. {pull}41061[41061]
+- Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206]
 
 *Auditbeat*