Merge branch 'main' into session-view-fix-missing-proc

elastic · Apr 30, 2024 · a0c1c21 · a0c1c21
2 parents 9b4cd6c + 5c684a8
commit a0c1c21
Show file tree

Hide file tree

Showing 10 changed files with 174 additions and 142 deletions.
diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml
@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          head-filter-regex: '(7\.1[6789]|8\.13|PR-.*|v8\.13\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'
@@ -28,11 +28,11 @@
               ignore-tags-older-than: -1
               ignore-tags-newer-than: 30
           - named-branches:
-              - regex-name:
-                  regex: '7\.1[6789]'
+              - exact-name:
+                  name: '8.13'
                   case-sensitive: true
               - regex-name:
-                  regex: '8\.\d+'
+                  regex: '7\.1[6789]'
                   case-sensitive: true
           - change-request:
               ignore-target-only-changes: true

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -12525,11 +12525,11 @@ various licenses:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.6.13
+Version: v0.6.14
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].13/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].14/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -25433,11 +25433,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
-Version: v0.21.0
+Version: v0.23.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.21.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.23.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 

diff --git a/go.mod b/go.mod
@@ -154,7 +154,7 @@ require (
 	golang.org/x/crypto v0.21.0
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
 	golang.org/x/mod v0.14.0
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.23.0
 	golang.org/x/oauth2 v0.10.0
 	golang.org/x/sync v0.5.0
 	golang.org/x/sys v0.18.0
@@ -203,7 +203,7 @@ require (
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.6.0
-	github.com/elastic/elastic-agent-autodiscover v0.6.13
+	github.com/elastic/elastic-agent-autodiscover v0.6.14
 	github.com/elastic/elastic-agent-libs v0.7.5
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.9.2

diff --git a/go.sum b/go.sum
@@ -551,8 +551,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ebpfevents v0.6.0 h1:BrL3m7JFK7U6h2jkbk3xAWWs//IZnugCHEDds5u2v68=
 github.com/elastic/ebpfevents v0.6.0/go.mod h1:ESG9gw7N+n5yCCMgdg1IIJENKWSmX7+X0Fi9GUs9nvU=
-github.com/elastic/elastic-agent-autodiscover v0.6.13 h1:zBeTxV+o2efEKntY+o6iMMNJ1AVjDXUqY3o6uzIkKaw=
-github.com/elastic/elastic-agent-autodiscover v0.6.13/go.mod h1:7P6YVKxuBT0qE/VxuA87obwZUAEU0O44mCN3r4/6x8w=
+github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP+WioV73ia17Et+UlFbz8=
+github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8=
 github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM=
 github.com/elastic/elastic-agent-client/v7 v7.8.1/go.mod h1:axl1nkdqc84YRFkeJGD9jExKNPUrOrzf3DFo2m653nY=
 github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg=
@@ -1960,8 +1960,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

diff --git a/libbeat/monitoring/report/log/log.go b/libbeat/monitoring/report/log/log.go
@@ -37,34 +37,36 @@ import (
 // TODO: Replace this with a proper solution that uses the metric type from
 // where it is defined. See: https://github.com/elastic/beats/issues/5433
 var gauges = map[string]bool{
-	"libbeat.output.events.active":       true,
-	"libbeat.pipeline.events.active":     true,
-	"libbeat.pipeline.clients":           true,
-	"libbeat.config.module.running":      true,
-	"registrar.states.current":           true,
-	"filebeat.events.active":             true,
-	"filebeat.harvester.running":         true,
-	"filebeat.harvester.open_files":      true,
-	"beat.memstats.memory_total":         true,
-	"beat.memstats.memory_alloc":         true,
-	"beat.memstats.rss":                  true,
-	"beat.memstats.gc_next":              true,
-	"beat.info.uptime.ms":                true,
-	"beat.cgroup.memory.mem.usage.bytes": true,
-	"beat.cpu.user.ticks":                true,
-	"beat.cpu.system.ticks":              true,
-	"beat.cpu.total.value":               true,
-	"beat.cpu.total.ticks":               true,
-	"beat.handles.open":                  true,
-	"beat.handles.limit.hard":            true,
-	"beat.handles.limit.soft":            true,
-	"beat.runtime.goroutines":            true,
-	"system.load.1":                      true,
-	"system.load.5":                      true,
-	"system.load.15":                     true,
-	"system.load.norm.1":                 true,
-	"system.load.norm.5":                 true,
-	"system.load.norm.15":                true,
+	"libbeat.output.events.active":             true,
+	"libbeat.pipeline.events.active":           true,
+	"libbeat.pipeline.clients":                 true,
+	"libbeat.pipeline.queue.max_events":        true,
+	"libbeat.pipeline.queue.filled.pct.events": true,
+	"libbeat.config.module.running":            true,
+	"registrar.states.current":                 true,
+	"filebeat.events.active":                   true,
+	"filebeat.harvester.running":               true,
+	"filebeat.harvester.open_files":            true,
+	"beat.memstats.memory_total":               true,
+	"beat.memstats.memory_alloc":               true,
+	"beat.memstats.rss":                        true,
+	"beat.memstats.gc_next":                    true,
+	"beat.info.uptime.ms":                      true,
+	"beat.cgroup.memory.mem.usage.bytes":       true,
+	"beat.cpu.user.ticks":                      true,
+	"beat.cpu.system.ticks":                    true,
+	"beat.cpu.total.value":                     true,
+	"beat.cpu.total.ticks":                     true,
+	"beat.handles.open":                        true,
+	"beat.handles.limit.hard":                  true,
+	"beat.handles.limit.soft":                  true,
+	"beat.runtime.goroutines":                  true,
+	"system.load.1":                            true,
+	"system.load.5":                            true,
+	"system.load.15":                           true,
+	"system.load.norm.1":                       true,
+	"system.load.norm.5":                       true,
+	"system.load.norm.15":                      true,
 }
 
 // isGauge returns true when the given metric key name represents a gauge value.
@@ -249,16 +251,16 @@ func toKeyValuePairs(snaps map[string]monitoring.FlatSnapshot) []interface{} {
 	for name, snap := range snaps {
 		data := make(mapstr.M, snapshotLen(snap))
 		for k, v := range snap.Bools {
-			data.Put(k, v) //nolint:errcheck // All keys within the flat snapshot are unique and are for scalar values.
+			data.Put(k, v)
 		}
 		for k, v := range snap.Floats {
-			data.Put(k, v) //nolint:errcheck // All keys within the flat snapshot are unique and are for scalar values.
+			data.Put(k, v)
 		}
 		for k, v := range snap.Ints {
-			data.Put(k, v) //nolint:errcheck // All keys within the flat snapshot are unique and are for scalar values.
+			data.Put(k, v)
 		}
 		for k, v := range snap.Strings {
-			data.Put(k, v) //nolint:errcheck // All keys within the flat snapshot are unique and are for scalar values.
+			data.Put(k, v)
 		}
 		if len(data) > 0 {
 			args = append(args, logp.Reflect(name, data))

diff --git a/libbeat/publisher/pipeline/monitoring.go b/libbeat/publisher/pipeline/monitoring.go
@@ -17,7 +17,11 @@
 
 package pipeline
 
-import "github.com/elastic/elastic-agent-libs/monitoring"
+import (
+	"math"
+
+	"github.com/elastic/elastic-agent-libs/monitoring"
+)
 
 type observer interface {
 	pipelineObserver
@@ -67,8 +71,9 @@ type metricsObserverVars struct {
 	activeEvents                        *monitoring.Uint
 
 	// queue metrics
-	queueACKed     *monitoring.Uint
-	queueMaxEvents *monitoring.Uint
+	queueACKed       *monitoring.Uint
+	queueMaxEvents   *monitoring.Uint
+	percentQueueFull *monitoring.Float
 }
 
 func newMetricsObserver(metrics *monitoring.Registry) *metricsObserver {
@@ -92,7 +97,8 @@ func newMetricsObserver(metrics *monitoring.Registry) *metricsObserver {
 			queueACKed:     monitoring.NewUint(reg, "queue.acked"),
 			queueMaxEvents: monitoring.NewUint(reg, "queue.max_events"),
 
-			activeEvents: monitoring.NewUint(reg, "events.active"), // Gauge
+			activeEvents:     monitoring.NewUint(reg, "events.active"), // Gauge
+			percentQueueFull: monitoring.NewFloat(reg, "queue.filled.pct.events"),
 		},
 	}
 }
@@ -121,12 +127,24 @@ func (o *metricsObserver) clientClosed() { o.vars.clients.Dec() }
 func (o *metricsObserver) newEvent() {
 	o.vars.events.Inc()
 	o.vars.activeEvents.Inc()
+	o.setPercentageFull()
+}
+
+// setPercentageFull is used interally to set the `queue.full` metric
+func (o *metricsObserver) setPercentageFull() {
+	maxEvt := o.vars.queueMaxEvents.Get()
+	if maxEvt != 0 {
+		pct := float64(o.vars.activeEvents.Get()) / float64(maxEvt)
+		pctRound := math.Round(pct/0.0005) * 0.0005
+		o.vars.percentQueueFull.Set(pctRound)
+	}
 }
 
 // (client) event is filtered out (on purpose or failed)
 func (o *metricsObserver) filteredEvent() {
 	o.vars.filtered.Inc()
 	o.vars.activeEvents.Dec()
+	o.setPercentageFull()
 }
 
 // (client) managed to push an event into the publisher pipeline
@@ -138,6 +156,7 @@ func (o *metricsObserver) publishedEvent() {
 func (o *metricsObserver) failedPublishEvent() {
 	o.vars.failed.Inc()
 	o.vars.activeEvents.Dec()
+	o.setPercentageFull()
 }
 
 //
@@ -148,11 +167,13 @@ func (o *metricsObserver) failedPublishEvent() {
 func (o *metricsObserver) queueACKed(n int) {
 	o.vars.queueACKed.Add(uint64(n))
 	o.vars.activeEvents.Sub(uint64(n))
+	o.setPercentageFull()
 }
 
 // (queue) maximum queue event capacity
 func (o *metricsObserver) queueMaxEvents(n int) {
 	o.vars.queueMaxEvents.Set(uint64(n))
+	o.setPercentageFull()
 }
 
 //

diff --git a/metricbeat/scripts/mage/package.go b/metricbeat/scripts/mage/package.go
@@ -40,7 +40,7 @@ const (
 // not supported. You must declare a dependency on either
 // PrepareModulePackagingOSS or PrepareModulePackagingXPack.
 func CustomizePackaging() {
-	mg.Deps(customizeLightModulesPackaging)
+	mg.Deps(CustomizeLightModulesPackaging)
 
 	var (
 		modulesDTarget = "modules.d"
@@ -104,7 +104,7 @@ func CustomizePackaging() {
 // PrepareModulePackagingOSS generates build/package/modules and
 // build/package/modules.d directories for use in packaging.
 func PrepareModulePackagingOSS() error {
-	err := prepareLightModulesPackaging("module")
+	err := PrepareLightModulesPackaging("module")
 	if err != nil {
 		return err
 	}
@@ -116,7 +116,7 @@ func PrepareModulePackagingOSS() error {
 // PrepareModulePackagingXPack generates build/package/modules and
 // build/package/modules.d directories for use in packaging.
 func PrepareModulePackagingXPack() error {
-	err := prepareLightModulesPackaging("module", devtools.OSSBeatDir("module"))
+	err := PrepareLightModulesPackaging("module", devtools.OSSBeatDir("module"))
 	if err != nil {
 		return err
 	}
@@ -201,8 +201,8 @@ func GenerateDirModulesD() error {
 	return nil
 }
 
-// customizeLightModulesPackaging customizes packaging to add light modules
-func customizeLightModulesPackaging() error {
+// CustomizeLightModulesPackaging customizes packaging to add light modules
+func CustomizeLightModulesPackaging() error {
 	var (
 		moduleTarget = "module"
 		module       = devtools.PackageFile{
@@ -225,8 +225,8 @@ func customizeLightModulesPackaging() error {
 	return nil
 }
 
-// prepareLightModulesPackaging generates light modules
-func prepareLightModulesPackaging(paths ...string) error {
+// PrepareLightModulesPackaging generates light modules
+func PrepareLightModulesPackaging(paths ...string) error {
 	err := devtools.Clean([]string{dirModulesGenerated})
 	if err != nil {
 		return err

diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go
@@ -20,6 +20,7 @@ import (
 
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
+	metricbeat "github.com/elastic/beats/v7/metricbeat/scripts/mage"
 	packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage"
 	osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage"
 
@@ -112,11 +113,19 @@ func CrossBuildDeps() error {
 	return callForBeat("crossBuildExt", "osquerybeat")
 }
 
+// PrepareLightModules prepares the module packaging.
+func PrepareLightModules() error {
+	return metricbeat.PrepareLightModulesPackaging(
+		filepath.Join("..", "metricbeat", "module"),       // x-pack/metricbeat
+		filepath.Join("..", "..", "metricbeat", "module"), // metricbeat (oss)
+	)
+}
+
 // Package packages the Beat for distribution.
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms.
 // Use VERSION_QUALIFIER to control the version qualifier.
-func Package() {
+func Package() error {
 	start := time.Now()
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
@@ -126,7 +135,14 @@ func Package() {
 	// Add osquery distro binaries, required for the osquerybeat subcommand.
 	osquerybeat.CustomizePackaging()
 
-	mg.SerialDeps(Update, osquerybeat.FetchOsqueryDistros, CrossBuildDeps, CrossBuild, devtools.Package, TestPackages)
+	// Add metricbeat lightweight modules.
+	if err := metricbeat.CustomizeLightModulesPackaging(); err != nil {
+		return err
+	}
+
+	mg.SerialDeps(Update, PrepareLightModules, osquerybeat.FetchOsqueryDistros, CrossBuildDeps, CrossBuild, devtools.Package, TestPackages)
+
+	return nil
 }
 
 // TestPackages tests the generated packages (i.e. file modes, owners, groups).