fix templates and docs to use correct -- command line args (#42060)

* fix templates and docs to use correct `--` command line args

Co-authored-by: Tiago Queiroz <[email protected]>

(cherry picked from commit 580f0f6)

CHANGELOG.next.asciidoc

@@ -13,6 +13,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix FQDN being lowercased when used as `host.hostname` {issue}39993[39993]
 - Beats won't log start up information when running under the Elastic Agent {40390}40390[40390]
 - Default Docker base image was reverted to Ubuntu 20.04 due to incompatability issues with glibc {pull}42144[42144]
+- Filebeat now needs `dup3`, `faccessat2`, `prctl` and `setrlimit` syscalls to run the journald input. If this input is not being used, the syscalls are not needed. All Beats have those syscalls allowed now because the default seccomp policy is global to all Beats. {pull}40061[40061]
+- Beats will rate limit the logs about errors when indexing events on Elasticsearch, logging a summary every 10s. The logs sent to the event log is unchanged. {issue}40157[40157]
+- Drop support for Debian 10 and upgrade statically linked glibc from 2.28 to 2.31 {pull}41402[41402]
+- Fix metrics not being ingested, due to "Limit of total fields [10000] has been exceeded while adding new fields [...]". The total fields limit has been increased to 12500. No significant performance impact on Elasticsearch is anticipated. {pull}41640[41640]
+- Set default kafka version to 2.1.0 in kafka output and filebeat. {pull}41662[41662]
+- Fix templates and docs to use correct `--` version of command line arguments. {issue}42038[42038] {pull}42060[42060]
 
 *Auditbeat*
 

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -295,7 +295,7 @@ spec:
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",
-          "-system.hostfs=/hostfs",
+          "--system.hostfs=/hostfs",
         ]
         env:
         - name: ELASTICSEARCH_HOST

deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml

@@ -25,7 +25,7 @@ spec:
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",
-          "-system.hostfs=/hostfs",
+          "--system.hostfs=/hostfs",
         ]
         env:
         - name: ELASTICSEARCH_HOST

dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml

@@ -298,7 +298,7 @@ spec:
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",
-          "-system.hostfs=/hostfs",
+          "--system.hostfs=/hostfs",
         ]
         env:
         - name: ELASTICSEARCH_HOST
@@ -403,7 +403,7 @@ spec:
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",
-          "-system.hostfs=/hostfs",
+          "--system.hostfs=/hostfs",
         ]
         ports:
         - containerPort: 56268

dev-tools/kubernetes/metricbeat/manifest.debug.yaml

@@ -298,7 +298,7 @@ spec:
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",
-          "-system.hostfs=/hostfs",
+          "--system.hostfs=/hostfs",
         ]
         ports:
         - containerPort: 56268

dev-tools/kubernetes/metricbeat/manifest.run.yaml

@@ -298,7 +298,7 @@ spec:
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",
-          "-system.hostfs=/hostfs",
+          "--system.hostfs=/hostfs",
         ]
         env:
         - name: ELASTICSEARCH_HOST

dev-tools/mage/pkg.go

@@ -172,7 +172,6 @@ func prepareIronbankBuild() error {
 		}
 		return nil
 	})
-
 	if err != nil {
 		return fmt.Errorf("cannot create templates for the IronBank: %w", err)
 	}
@@ -206,7 +205,7 @@ func saveIronbank() error {
 
 	distributionsDir := "build/distributions"
 	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
-		err := os.MkdirAll(distributionsDir, 0750)
+		err := os.MkdirAll(distributionsDir, 0o750)
 		if err != nil {
 			return fmt.Errorf("cannot create folder for docker artifacts: %w", err)
 		}
@@ -348,9 +347,7 @@ func TestPackages(options ...TestPackagesOption) error {
 	args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*"))
 
 	if out, err := goTest(args...); err != nil {
-		if !mg.Verbose() {
-			fmt.Println(out)
-		}
+		fmt.Println(out)
 		return err
 	}
 

dev-tools/packaging/package_test.go

@@ -26,7 +26,9 @@ import (
 	"bufio"
 	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -36,18 +38,20 @@ import (
 	"slices"
 	"strings"
 	"testing"
-
-	"errors"
+	"time"
 
 	"github.com/blakesmith/ar"
 	rpm "github.com/cavaliergopher/rpm"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/docker/client"
 )
 
 const (
-	expectedConfigMode     = os.FileMode(0600)
-	expectedManifestMode   = os.FileMode(0644)
+	expectedConfigMode     = os.FileMode(0o600)
+	expectedManifestMode   = os.FileMode(0o644)
 	expectedModuleFileMode = expectedManifestMode
-	expectedModuleDirMode  = os.FileMode(0755)
+	expectedModuleDirMode  = os.FileMode(0o755)
 )
 
 var (
@@ -234,15 +238,15 @@ func checkDocker(t *testing.T, file string) {
 		t.Errorf("error reading file %v: %v", file, err)
 		return
 	}
-
 	checkDockerEntryPoint(t, p, info)
 	checkDockerLabels(t, p, info, file)
 	checkDockerUser(t, p, info, *rootUserContainer)
-	checkConfigPermissionsWithMode(t, p, os.FileMode(0644))
-	checkManifestPermissionsWithMode(t, p, os.FileMode(0644))
+	checkConfigPermissionsWithMode(t, p, os.FileMode(0o644))
+	checkManifestPermissionsWithMode(t, p, os.FileMode(0o644))
 	checkModulesPresent(t, "", p)
 	checkModulesDPresent(t, "", p)
 	checkLicensesPresent(t, "licenses/", p)
+	checkDockerImageRun(t, p, file)
 }
 
 // Verify that the main configuration file is installed with a 0600 file mode.
@@ -356,7 +360,7 @@ func checkModulesOwner(t *testing.T, p *packageFile, expectRoot bool) {
 // Verify that the systemd unit file has a mode of 0644. It should not be
 // executable.
 func checkSystemdUnitPermissions(t *testing.T, p *packageFile) {
-	const expectedMode = os.FileMode(0644)
+	const expectedMode = os.FileMode(0o644)
 	t.Run(p.Name+" systemd unit file permissions", func(t *testing.T) {
 		for _, entry := range p.Contents {
 			if systemdUnitFilePattern.MatchString(entry.File) {
@@ -443,7 +447,7 @@ func checkLicensesPresent(t *testing.T, prefix string, p *packageFile) {
 }
 
 func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) {
-	expectedMode := os.FileMode(0755)
+	expectedMode := os.FileMode(0o755)
 
 	t.Run(fmt.Sprintf("%s entrypoint", p.Name), func(t *testing.T) {
 		if len(info.Config.Entrypoint) == 0 {
@@ -511,6 +515,111 @@ func checkDockerUser(t *testing.T, p *packageFile, info *dockerInfo, expectRoot
 	})
 }
 
+func checkDockerImageRun(t *testing.T, p *packageFile, imagePath string) {
+	t.Run(fmt.Sprintf("%s check docker images runs", p.Name), func(t *testing.T) {
+		var ctx context.Context
+		dl, ok := t.Deadline()
+		if !ok {
+			ctx = context.Background()
+		} else {
+			c, cancel := context.WithDeadline(context.Background(), dl)
+			ctx = c
+			defer cancel()
+		}
+		f, err := os.Open(imagePath)
+		if err != nil {
+			t.Fatalf("failed to open docker image %q: %s", imagePath, err)
+		}
+		defer f.Close()
+
+		c, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		if err != nil {
+			t.Fatalf("failed to get a Docker client: %s", err)
+		}
+
+		loadResp, err := c.ImageLoad(ctx, f, true)
+		if err != nil {
+			t.Fatalf("error loading docker image: %s", err)
+		}
+
+		loadRespBody, err := io.ReadAll(loadResp.Body)
+		if err != nil {
+			t.Fatalf("failed to read image load response: %s", err)
+		}
+		loadResp.Body.Close()
+
+		_, after, found := strings.Cut(string(loadRespBody), "Loaded image: ")
+		if !found {
+			t.Fatalf("image load response was unexpected: %s", string(loadRespBody))
+		}
+		imageId := strings.TrimRight(after, "\\n\"}\r\n")
+
+		var caps strslice.StrSlice
+		if strings.Contains(imageId, "packetbeat") {
+			caps = append(caps, "NET_ADMIN")
+		}
+
+		createResp, err := c.ContainerCreate(ctx,
+			&container.Config{
+				Image: imageId,
+			},
+			&container.HostConfig{
+				CapAdd: caps,
+			},
+			nil,
+			nil,
+			"")
+		if err != nil {
+			t.Fatalf("error creating container from image: %s", err)
+		}
+		defer func() {
+			err := c.ContainerRemove(ctx, createResp.ID, container.RemoveOptions{Force: true})
+			if err != nil {
+				t.Errorf("error removing container: %s", err)
+			}
+		}()
+
+		err = c.ContainerStart(ctx, createResp.ID, container.StartOptions{})
+		if err != nil {
+			t.Fatalf("failed to start container: %s", err)
+		}
+		defer func() {
+			err := c.ContainerStop(ctx, createResp.ID, container.StopOptions{})
+			if err != nil {
+				t.Errorf("error stopping container: %s", err)
+			}
+		}()
+
+		timer := time.NewTimer(15 * time.Second)
+		defer timer.Stop()
+		ticker := time.NewTicker(2 * time.Second)
+		defer ticker.Stop()
+
+		var logs []byte
+		sentinelLog := "Beat ID: "
+		for {
+			select {
+			case <-timer.C:
+				t.Fatalf("never saw %q within timeout\nlogs:\n%s", sentinelLog, string(logs))
+				return
+			case <-ticker.C:
+				out, err := c.ContainerLogs(ctx, createResp.ID, container.LogsOptions{ShowStdout: true, ShowStderr: true})
+				if err != nil {
+					t.Logf("could not get logs: %s", err)
+				}
+				logs, err = io.ReadAll(out)
+				out.Close()
+				if err != nil {
+					t.Logf("error reading logs: %s", err)
+				}
+				if bytes.Contains(logs, []byte(sentinelLog)) {
+					return
+				}
+			}
+		}
+	})
+}
+
 // ensureNoBuildIDLinks checks for regressions related to
 // https://github.com/elastic/beats/issues/12956.
 func ensureNoBuildIDLinks(t *testing.T, p *packageFile) {

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -243,4 +243,4 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/
 
 WORKDIR {{ $beatHome }}
 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint"]
-CMD ["-environment", "container"]
+CMD ["--environment", "container"]

dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile

@@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT}
 
 # TODO: eventually /tinit will be replaced by /usr/bin/tini
 ENTRYPOINT ["/tinit", "--", "/usr/share/auditbeat/auditbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/auditbeat/data/auditbeat.sock"]
-CMD ["-environment", "container"]
+CMD ["--environment", "container"]
 
 # see https://www.elastic.co/guide/en/beats/auditbeat/current/http-endpoint.html
 HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/auditbeat/data/auditbeat.sock' 'http:/stats?pretty'

dev-tools/packaging/templates/ironbank/filebeat/Dockerfile

@@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT}
 
 # TODO: eventually /tinit will be replaced by /usr/bin/tini
 ENTRYPOINT ["/tinit", "--", "/usr/share/filebeat/filebeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/filebeat/data/filebeat.sock"]
-CMD ["-environment", "container"]
+CMD ["--environment", "container"]
 
 # see https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html
 HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/filebeat/data/filebeat.sock' 'http:/stats/?pretty'

dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile

@@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT}
 
 # TODO: eventually /tinit will be replaced by /usr/bin/tini
 ENTRYPOINT ["/tinit", "--", "/usr/share/heartbeat/heartbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/heartbeat/data/heartbeat.sock"]
-CMD ["-environment", "container"]
+CMD ["--environment", "container"]
 
 # see https://www.elastic.co/guide/en/beats/heartbeat/current/http-endpoint.html
 HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/heartbeat/data/heartbeat.sock' 'http:/stats/?pretty'

dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile

@@ -81,7 +81,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT}
 
 # TODO: eventually /tinit will be replaced by /usr/bin/tini
 ENTRYPOINT ["/tinit", "--", "/usr/share/metricbeat/metricbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/metricbeat/data/metricbeat.sock"]
-CMD ["-environment", "container"]
+CMD ["--environment", "container"]
 
 # see https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html
 HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/metricbeat/data/metricbeat.sock' 'http:/stats/?pretty'

dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile

@@ -82,7 +82,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT}
 
 # TODO: eventually /tinit will be replaced by /usr/bin/tini
 ENTRYPOINT ["/tinit", "--", "/usr/share/packetbeat/packetbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/packetbeat/data/packetbeat.sock"]
-CMD ["-environment", "container"]
+CMD ["--environment", "container"]
 
 # see https://www.elastic.co/guide/en/beats/packetbeat/current/http-endpoint.html
 HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/packetbeat/data/packetbeat.sock' 'http:/stats/?pretty'

filebeat/tests/open-file-handlers/run.sh

@@ -1,3 +1,3 @@
 /etc/init.d/metricbeat start
 cd /filebeat
-./filebeat -httpprof :6060
+./filebeat --httpprof :6060

libbeat/docs/command-reference.asciidoc

@@ -1064,7 +1064,7 @@ messages.
 *`-e, --e`*::
 Logs to stderr and disables syslog/file output.
 
-*`-environment`*::
+*`--environment`*::
 For logging purposes, specifies the environment that {beatname_uc} is running in.
 This setting is used to select a default log output when no log output is configured.
 Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`.

metricbeat/docs/running-on-docker.asciidoc

@@ -22,7 +22,7 @@ docker run \
   --env DBUS_SYSTEM_BUS_ADDRESS='unix:path=/hostfs/var/run/dbus/system_bus_socket' \ <4>
   --net=host \ <5>
   --cgroupns=host \ <6>
-  {dockerimage} -e -system.hostfs=/hostfs
+  {dockerimage} -e --system.hostfs=/hostfs
 ----
 
 <1> Metricbeat's <<metricbeat-module-system,system module>> collects much of its data through the Linux proc

metricbeat/module/kubernetes/_meta/remote-debugger/README.md

@@ -72,7 +72,7 @@ containers:
   args: [
     "-c", "/etc/metricbeat.yml",
     "-e",
-    "-system.hostfs=/hostfs",
+    "--system.hostfs=/hostfs",
   ]
   ports:
     - containerPort: 56268