Merge branch 'main' into module_pgbouncer

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -6,7 +6,7 @@ env:
   MODULE: "kubernetes"
 
   # Other deps
-  ASDF_KIND_VERSION: "0.20.0"
+  ASDF_KIND_VERSION: "0.24.0"
 
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 

.buildkite/filebeat/filebeat-pipeline.yml

@@ -24,7 +24,7 @@ env:
   ASDF_MAGE_VERSION: 1.15.0
 
   # Integration Tests
-  K8S_VERSION: "v1.29.0"
+  K8S_VERSION: "v1.31.0"
   ASDF_KIND_VERSION: "0.20.0"
 
   # Unit tests

.buildkite/metricbeat/pipeline.yml

@@ -21,7 +21,7 @@ env:
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
   # Integration Tests
-  K8S_VERSION: "v1.29.0"
+  K8S_VERSION: "v1.31.0"
   ASDF_KIND_VERSION: "0.20.0"
 
   # Module Tests

.github/CODEOWNERS

@@ -12,14 +12,18 @@
 # https://github.community/t/codeowners-file-with-a-not-file-type-condition/1423/9
 CHANGELOG*
 
+# Top-level files ownership
+/catalog-info.yaml @elastic/ingest-eng-prod
 # The tech leads of the teams working in Beats share ownership of the Go module dependencies and related files.
-/.github/CODEOWNERS @elastic/beats-tech-leads
 /go.mod @elastic/beats-tech-leads
 /go.sum @elastic/beats-tech-leads
 /NOTICE.txt @elastic/beats-tech-leads
 
+# Sub-directories/files ownership. Remember that order matters; the last matching pattern takes the most precedence.
+/.buildkite @elastic/ingest-eng-prod
 /.ci/ @elastic/elastic-agent-data-plane @elastic/ingest-eng-prod
 /.github/ @elastic/ingest-eng-prod
+/.github/CODEOWNERS @elastic/beats-tech-leads
 /auditbeat/ @elastic/sec-linux-platform
 /deploy/ @elastic/elastic-agent-data-plane
 /deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/obs-cloudnative-monitoring
@@ -55,6 +59,8 @@ CHANGELOG*
 /libbeat/ @elastic/elastic-agent-data-plane
 /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs
 /libbeat/management @elastic/elastic-agent-control-plane
+/libbeat/processors/add_cloud_metadata @elastic/obs-cloud-monitoring
+/libbeat/processors/add_kubernetes_metadata @elastic/obs-cloudnative-monitoring
 /libbeat/processors/cache/ @elastic/security-service-integrations
 /libbeat/processors/community_id/ @elastic/sec-deployment-and-devices
 /libbeat/processors/decode_xml/ @elastic/security-service-integrations
@@ -63,9 +69,8 @@ CHANGELOG*
 /libbeat/processors/registered_domain/ @elastic/sec-deployment-and-devices
 /libbeat/processors/syslog/ @elastic/sec-deployment-and-devices
 /libbeat/processors/translate_sid/ @elastic/sec-windows-platform
-/libbeat/processors/add_cloud_metadata @elastic/obs-cloud-monitoring
-/libbeat/processors/add_kubernetes_metadata @elastic/obs-cloudnative-monitoring
 /libbeat/reader/syslog/ @elastic/sec-deployment-and-devices
+/libbeat/scripts @elastic/ingest-eng-prod
 /licenses/ @elastic/elastic-agent-data-plane
 /metricbeat/ @elastic/elastic-agent-data-plane
 /metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
@@ -99,6 +104,7 @@ CHANGELOG*
 /metricbeat/module/system/ @elastic/elastic-agent-data-plane
 /metricbeat/module/vsphere @elastic/obs-infraobs-integrations
 /metricbeat/module/zookeeper @elastic/obs-infraobs-integrations
+/metricbeat/tests @elastic/ingest-eng-prod
 /packetbeat/ @elastic/sec-linux-platform
 /script/ @elastic/elastic-agent-data-plane
 /testing/ @elastic/elastic-agent-data-plane
@@ -227,9 +233,3 @@ CHANGELOG*
 /x-pack/osquerybeat/ @elastic/sec-deployment-and-devices
 /x-pack/packetbeat/ @elastic/sec-linux-platform
 /x-pack/winlogbeat/ @elastic/sec-windows-platform
-
-# Ownership of CI or related files by the Ingest Eng Prod team
-/.buildkite @elastic/ingest-eng-prod
-/catalog-info.yaml @elastic/ingest-eng-prod
-/libbeat/scripts @elastic/ingest-eng-prod
-/metricbeat/tests @elastic/ingest-eng-prod

.mergify.yml

@@ -355,6 +355,19 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.16 branch
+    conditions:
+      - merged
+      - label~=^(backport-v8.16.0|backport-8.16)$
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.16"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 8.x branch
     conditions:
       - merged

CHANGELOG-developer.next.asciidoc

@@ -70,6 +70,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Debug log entries from the acker (`stateful ack ...` or `stateless ack ...`) removed. {pull}39672[39672]
 - Rename x-pack/filebeat websocket input to streaming. {issue}40264[40264] {pull}40421[40421]
 - Journald input now calls `journalctl` instead of using `github.com/coreos/go-systemd/[email protected]/sdjournal`, the CGO dependency has been removed from Filebeat {pull}40061[40061]
+- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files, however the ingest pipeline sets it back to the original input (log or journald). {pull}41246[41246]
 
 ==== Bugfixes
 
@@ -208,6 +209,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] 
 - Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936]
 - Add field redaction package. {pull}40997[40997]
+- Add support for marked redaction to x-pack/filebeat/input/internal/private {pull}41212[41212]
+- Add support for collecting Okta role and factor data for users with filebeat entityanalytics input. {pull}41044[41044]
 
 ==== Deprecated
 

CHANGELOG.asciidoc

@@ -3,6 +3,47 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.15.3]]
+=== Beats version 8.15.3
+https://github.com/elastic/beats/compare/v8.15.2\...v8.15.3[View commits]
+
+==== Breaking changes
+
+*Filebeat*
+
+- Change `log.file.path` field in awscloudwatch input to nested object. {pull}41099[41099]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Allow port number 0 in the community ID flowhash processor. {pull}40259[40259]
+- The journald input now restarts if there is an error/crash. {issue}32782[32782] {pull}40558[40558]
+
+*Filebeat*
+
+- Fix replace processor handling of zero string replacement validation. {pull}40751[40751]
+- Add backup and delete for AWS S3 polling mode feature back. {pull}41071[41071]
+
+*Metricbeat*
+
+- Use namespace for GetListMetrics when it exists in AWS. {pull}41022[41022]
+
+*Packetbeat*
+
+- Fix upload of bundled ingest pipelines on Windows. {pull}41110[41110]
+
+==== Added
+
+*Affecting all Beats*
+
+- Update Go version to 1.22.8. {pull}41139[41139]
+
+*Metricbeat*
+
+- Restore `docker.network.in.*` and `docker.network.out.*` fields in docker module. {pull}40968[40968]
+
+
 [[release-notes-8.15.2]]
 === Beats version 8.15.2
 https://github.com/elastic/beats/compare/v8.15.0\...v8.15.2[View commits]

CHANGELOG.next.asciidoc

@@ -46,6 +46,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added `container.image.name` to `journald` Filebeat input's Docker-specific translated fields. {pull}40450[40450]
 - Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099]
 - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089]
+- The performance of ingesting SQS data with the S3 input has improved by up to 60x for queues with many small events. `max_number_of_messages` config for SQS mode is now ignored, as the new design no longer needs a manual cap on messages. Instead, use `number_of_workers` to scale ingestion rate in both S3 and SQS modes. The increased efficiency may increase network bandwidth consumption, which can be throttled by lowering `number_of_workers`. It may also increase number of events stored in memory, which can be throttled by lowering the configured size of the internal queue. {pull}40699[40699]
+- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files. {pull}41061[41061]
 
 
 *Heartbeat*
@@ -104,23 +106,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Support Elastic Agent control protocol chunking support {pull}37343[37343]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
 - Set timeout of 1 minute for FQDN requests {pull}37756[37756]
-- Fix the paths in the .cmd script added to the path by the Windows MSI to point to the new C:\Program Files installation location. https://github.com/elastic/elastic-stack-installers/pull/238
-- Change cache processor documentation from `write_period` to `write_interval`. {pull}38561[38561]
-- Fix cache processor expiries heap cleanup on partial file writes. {pull}38561[38561]
-- Fix cache processor expiries infinite growth when large a large TTL is used and recurring keys are cached. {pull}38561[38561]
-- Fix parsing of RFC 3164 process IDs in syslog processor. {issue}38947[38947] {pull}38982[38982]
-- Rename the field "apache2.module.error" to "apache.module.error" in Apache error visualization. {issue}39480[39480] {pull}39481[39481]
-- Validate config of the `replace` processor {pull}40047[40047]
-- Allow port number 0 in the community ID flowhash processor {pull}40259[40259]
-- Fix handling of escaped brackets in syslog structured data. {issue}40445[40445] {pull}40446[40446]
-- Aborts all active connections for Elasticsearch output. {pull}40572[40572]
-- Closes beat Publisher on beat stop and by the Agent manager. {pull}40572[40572]
-- The journald input now restarts if there is an error/crash {issue}32782[32782] {pull}40558[40558]
-- Update Go version to 1.22.8. {pull}41139[41139]
 
 *Auditbeat*
 
-
+- Request status from a separate socket to avoid data congestion {pull}41207[41207]
 
 *Filebeat*
 
@@ -174,6 +163,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix a bug in Salesforce input to only handle responses with 200 status code {pull}41015[41015]
 - Fixed failed job handling and removed false-positive error logs in the GCS input. {pull}41142[41142]
 - Bump github.com/elastic/go-sfdc dependency used by x-pack/filebeat/input/salesforce. {pull}41192[41192]
+- Log bad handshake details when websocket connection fails {pull}41300[41300]
 
 *Heartbeat*
 
@@ -205,13 +195,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add AWS OwningAccount support for cross account monitoring {issue}40570[40570] {pull}40691[40691]
 - Use namespace for GetListMetrics when exists in AWS {pull}41022[41022]
 - Fix http server helper SSL config. {pull}39405[39405]
+- Fix Kubernetes metadata sometimes not being present after startup {pull}41216[41216]
 
 *Osquerybeat*
 
 
 *Packetbeat*
 
-- Fix upload of bundled ingest pipelines on Windows. {pull}41110[41110]
 
 *Winlogbeat*
 
@@ -236,7 +226,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - The script processor has a new configuration option that only uses the cached javascript sessions and prevents the creation of new javascript sessions.
 - Update to Go 1.22.7. {pull}41018[41018]
 - Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942]
-
+- Reduce memory consumption of k8s autodiscovery and the add_kubernetes_metadata processor when Deployment metadata is enabled
 
 *Auditbeat*
 
@@ -324,6 +314,11 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Improved GCS input documentation. {pull}41143[41143]
 - Add CSV decoding capacity to azureblobstorage input {pull}40978[40978]
 - Add CSV decoding capacity to gcs input {pull}40979[40979]
+- Add support to source AWS cloudwatch logs from linked accounts. {pull}41188[41188]
+- Jounrald input now supports filtering by facilities {pull}41061[41061]
+- System module now supports reading from jounrald. {pull}41061[41061]
+- Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206]
+- Improved Azure Blob Storage input documentation. {pull}41252[41252]
 
 *Auditbeat*
 
@@ -370,6 +365,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added Cisco Meraki module {pull}40836[40836]
 - Added Palo Alto Networks module {pull}40686[40686]
 - Restore docker.network.in.* and docker.network.out.* fields in docker module {pull}40968[40968]
+- Only watch metadata for ReplicaSets in metricbeat k8s module {pull}41289[41289]
 
 *Metricbeat*
 
@@ -423,3 +419,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 
 
+
+
+
+
+
+