diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f158821dffc7..4d81db1b1fdb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -13,6 +13,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix FQDN being lowercased when used as `host.hostname` {issue}39993[39993] - Beats won't log start up information when running under the Elastic Agent {40390}40390[40390] - Default Docker base image was reverted to Ubuntu 20.04 due to incompatability issues with glibc {pull}42144[42144] +- Filebeat now needs `dup3`, `faccessat2`, `prctl` and `setrlimit` syscalls to run the journald input. If this input is not being used, the syscalls are not needed. All Beats have those syscalls allowed now because the default seccomp policy is global to all Beats. {pull}40061[40061] +- Beats will rate limit the logs about errors when indexing events on Elasticsearch, logging a summary every 10s. The logs sent to the event log is unchanged. {issue}40157[40157] +- Drop support for Debian 10 and upgrade statically linked glibc from 2.28 to 2.31 {pull}41402[41402] +- Fix metrics not being ingested, due to "Limit of total fields [10000] has been exceeded while adding new fields [...]". The total fields limit has been increased to 12500. No significant performance impact on Elasticsearch is anticipated. {pull}41640[41640] +- Set default kafka version to 2.1.0 in kafka output and filebeat. {pull}41662[41662] +- Fix templates and docs to use correct `--` version of command line arguments. {issue}42038[42038] {pull}42060[42060] *Auditbeat* diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 53ddd3ab2f6d..ebe2cf2eb017 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -295,7 +295,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml index e8c0074be6de..c89dd2b21d39 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml @@ -25,7 +25,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml index 6dd492804c8b..c266335babad 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml @@ -298,7 +298,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST @@ -403,7 +403,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] ports: - containerPort: 56268 diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml index 398d7fa85606..7bb9dd963366 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.debug.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml @@ -298,7 +298,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] ports: - containerPort: 56268 diff --git a/dev-tools/kubernetes/metricbeat/manifest.run.yaml b/dev-tools/kubernetes/metricbeat/manifest.run.yaml index 21c9727d45ef..6b42510754e8 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.run.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.run.yaml @@ -298,7 +298,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 53783b29553a..757f857265f4 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -172,7 +172,6 @@ func prepareIronbankBuild() error { } return nil }) - if err != nil { return fmt.Errorf("cannot create templates for the IronBank: %w", err) } @@ -206,7 +205,7 @@ func saveIronbank() error { distributionsDir := "build/distributions" if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { - err := os.MkdirAll(distributionsDir, 0750) + err := os.MkdirAll(distributionsDir, 0o750) if err != nil { return fmt.Errorf("cannot create folder for docker artifacts: %w", err) } @@ -348,9 +347,7 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*")) if out, err := goTest(args...); err != nil { - if !mg.Verbose() { - fmt.Println(out) - } + fmt.Println(out) return err } diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index cad23aec9e87..6a9a72a8facd 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -26,7 +26,9 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -36,18 +38,20 @@ import ( "slices" "strings" "testing" - - "errors" + "time" "github.com/blakesmith/ar" rpm "github.com/cavaliergopher/rpm" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/client" ) const ( - expectedConfigMode = os.FileMode(0600) - expectedManifestMode = os.FileMode(0644) + expectedConfigMode = os.FileMode(0o600) + expectedManifestMode = os.FileMode(0o644) expectedModuleFileMode = expectedManifestMode - expectedModuleDirMode = os.FileMode(0755) + expectedModuleDirMode = os.FileMode(0o755) ) var ( @@ -234,15 +238,15 @@ func checkDocker(t *testing.T, file string) { t.Errorf("error reading file %v: %v", file, err) return } - checkDockerEntryPoint(t, p, info) checkDockerLabels(t, p, info, file) checkDockerUser(t, p, info, *rootUserContainer) - checkConfigPermissionsWithMode(t, p, os.FileMode(0644)) - checkManifestPermissionsWithMode(t, p, os.FileMode(0644)) + checkConfigPermissionsWithMode(t, p, os.FileMode(0o644)) + checkManifestPermissionsWithMode(t, p, os.FileMode(0o644)) checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) checkLicensesPresent(t, "licenses/", p) + checkDockerImageRun(t, p, file) } // Verify that the main configuration file is installed with a 0600 file mode. @@ -356,7 +360,7 @@ func checkModulesOwner(t *testing.T, p *packageFile, expectRoot bool) { // Verify that the systemd unit file has a mode of 0644. It should not be // executable. func checkSystemdUnitPermissions(t *testing.T, p *packageFile) { - const expectedMode = os.FileMode(0644) + const expectedMode = os.FileMode(0o644) t.Run(p.Name+" systemd unit file permissions", func(t *testing.T) { for _, entry := range p.Contents { if systemdUnitFilePattern.MatchString(entry.File) { @@ -443,7 +447,7 @@ func checkLicensesPresent(t *testing.T, prefix string, p *packageFile) { } func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) { - expectedMode := os.FileMode(0755) + expectedMode := os.FileMode(0o755) t.Run(fmt.Sprintf("%s entrypoint", p.Name), func(t *testing.T) { if len(info.Config.Entrypoint) == 0 { @@ -511,6 +515,111 @@ func checkDockerUser(t *testing.T, p *packageFile, info *dockerInfo, expectRoot }) } +func checkDockerImageRun(t *testing.T, p *packageFile, imagePath string) { + t.Run(fmt.Sprintf("%s check docker images runs", p.Name), func(t *testing.T) { + var ctx context.Context + dl, ok := t.Deadline() + if !ok { + ctx = context.Background() + } else { + c, cancel := context.WithDeadline(context.Background(), dl) + ctx = c + defer cancel() + } + f, err := os.Open(imagePath) + if err != nil { + t.Fatalf("failed to open docker image %q: %s", imagePath, err) + } + defer f.Close() + + c, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + t.Fatalf("failed to get a Docker client: %s", err) + } + + loadResp, err := c.ImageLoad(ctx, f, true) + if err != nil { + t.Fatalf("error loading docker image: %s", err) + } + + loadRespBody, err := io.ReadAll(loadResp.Body) + if err != nil { + t.Fatalf("failed to read image load response: %s", err) + } + loadResp.Body.Close() + + _, after, found := strings.Cut(string(loadRespBody), "Loaded image: ") + if !found { + t.Fatalf("image load response was unexpected: %s", string(loadRespBody)) + } + imageId := strings.TrimRight(after, "\\n\"}\r\n") + + var caps strslice.StrSlice + if strings.Contains(imageId, "packetbeat") { + caps = append(caps, "NET_ADMIN") + } + + createResp, err := c.ContainerCreate(ctx, + &container.Config{ + Image: imageId, + }, + &container.HostConfig{ + CapAdd: caps, + }, + nil, + nil, + "") + if err != nil { + t.Fatalf("error creating container from image: %s", err) + } + defer func() { + err := c.ContainerRemove(ctx, createResp.ID, container.RemoveOptions{Force: true}) + if err != nil { + t.Errorf("error removing container: %s", err) + } + }() + + err = c.ContainerStart(ctx, createResp.ID, container.StartOptions{}) + if err != nil { + t.Fatalf("failed to start container: %s", err) + } + defer func() { + err := c.ContainerStop(ctx, createResp.ID, container.StopOptions{}) + if err != nil { + t.Errorf("error stopping container: %s", err) + } + }() + + timer := time.NewTimer(15 * time.Second) + defer timer.Stop() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + var logs []byte + sentinelLog := "Beat ID: " + for { + select { + case <-timer.C: + t.Fatalf("never saw %q within timeout\nlogs:\n%s", sentinelLog, string(logs)) + return + case <-ticker.C: + out, err := c.ContainerLogs(ctx, createResp.ID, container.LogsOptions{ShowStdout: true, ShowStderr: true}) + if err != nil { + t.Logf("could not get logs: %s", err) + } + logs, err = io.ReadAll(out) + out.Close() + if err != nil { + t.Logf("error reading logs: %s", err) + } + if bytes.Contains(logs, []byte(sentinelLog)) { + return + } + } + } + }) +} + // ensureNoBuildIDLinks checks for regressions related to // https://github.com/elastic/beats/issues/12956. func ensureNoBuildIDLinks(t *testing.T, p *packageFile) { diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 80e034dfae8c..8b80a36ad9a1 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -243,4 +243,4 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/ WORKDIR {{ $beatHome }} ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] diff --git a/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile index 893e84d57e36..387b1c4f4f14 100644 --- a/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile @@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/auditbeat/auditbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/auditbeat/data/auditbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/auditbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/auditbeat/data/auditbeat.sock' 'http:/stats?pretty' diff --git a/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile b/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile index dc4f7bb49ea9..5f8b5fafb652 100644 --- a/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile @@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/filebeat/filebeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/filebeat/data/filebeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/filebeat/data/filebeat.sock' 'http:/stats/?pretty' diff --git a/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile index c71357b8478e..b24ce932c3cf 100644 --- a/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile @@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/heartbeat/heartbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/heartbeat/data/heartbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/heartbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/heartbeat/data/heartbeat.sock' 'http:/stats/?pretty' diff --git a/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile index cee8fe1fe6fa..e0f832d1ef57 100644 --- a/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile @@ -81,7 +81,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/metricbeat/metricbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/metricbeat/data/metricbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/metricbeat/data/metricbeat.sock' 'http:/stats/?pretty' diff --git a/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile index e7a025ef6aa0..6d7155c80c06 100644 --- a/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile @@ -82,7 +82,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/packetbeat/packetbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/packetbeat/data/packetbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/packetbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/packetbeat/data/packetbeat.sock' 'http:/stats/?pretty' diff --git a/filebeat/tests/open-file-handlers/run.sh b/filebeat/tests/open-file-handlers/run.sh index 4554660ab2f8..1bcc545391d0 100644 --- a/filebeat/tests/open-file-handlers/run.sh +++ b/filebeat/tests/open-file-handlers/run.sh @@ -1,3 +1,3 @@ /etc/init.d/metricbeat start cd /filebeat -./filebeat -httpprof :6060 +./filebeat --httpprof :6060 diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index 91daaf097be6..b954faad2cc4 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -1064,7 +1064,7 @@ messages. *`-e, --e`*:: Logs to stderr and disables syslog/file output. -*`-environment`*:: +*`--environment`*:: For logging purposes, specifies the environment that {beatname_uc} is running in. This setting is used to select a default log output when no log output is configured. Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`. diff --git a/metricbeat/docs/running-on-docker.asciidoc b/metricbeat/docs/running-on-docker.asciidoc index 26ba27658b8f..da012b2eb556 100644 --- a/metricbeat/docs/running-on-docker.asciidoc +++ b/metricbeat/docs/running-on-docker.asciidoc @@ -22,7 +22,7 @@ docker run \ --env DBUS_SYSTEM_BUS_ADDRESS='unix:path=/hostfs/var/run/dbus/system_bus_socket' \ <4> --net=host \ <5> --cgroupns=host \ <6> - {dockerimage} -e -system.hostfs=/hostfs + {dockerimage} -e --system.hostfs=/hostfs ---- <1> Metricbeat's <> collects much of its data through the Linux proc diff --git a/metricbeat/module/kubernetes/_meta/remote-debugger/README.md b/metricbeat/module/kubernetes/_meta/remote-debugger/README.md index cdebbe225c2a..a62eca2a051b 100644 --- a/metricbeat/module/kubernetes/_meta/remote-debugger/README.md +++ b/metricbeat/module/kubernetes/_meta/remote-debugger/README.md @@ -72,7 +72,7 @@ containers: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] ports: - containerPort: 56268