tetragon: Add missed stats to kprobemetrics package

Add tetragon_missed_probes_total metric to kprobemetrics package and
logic to store and collect missed stats.

The missed stats are supported for kprobes and stored per function
and policy name, like for generic kprobes:

  tetragon_missed_probes_total{attach="__x64_sys_close",policy="syswritefollowfdpsswd"} 453
  tetragon_missed_probes_total{attach="__x64_sys_write",policy="syswritefollowfdpsswd"} 455
  tetragon_missed_probes_total{attach="fd_install",policy="syswritefollowfdpsswd"} 451

and multi kprobes:

  tetragon_missed_probes_total{attach="kprobe_multi (3 functions)",policy="syswritefollowfdpsswd"} 41

Signed-off-by: Jiri Olsa <[email protected]>

pkg/metrics/kprobemetrics/collector.go

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+package kprobemetrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// bpfCollector implements prometheus.Collector. It collects metrics directly from BPF maps.
+type bpfCollector struct{}
+
+func NewBPFCollector() prometheus.Collector {
+	return &bpfCollector{}
+}
+
+func (c *bpfCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- MissedProbes.Desc()
+}
+
+func (c *bpfCollector) Collect(ch chan<- prometheus.Metric) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	for key, stat := range missedStats {
+		ch <- MissedProbes.MustMetric(stat.missed, stat.policy, key.attach)
+	}
+}

pkg/metrics/kprobemetrics/kprobemetrics.go

@@ -52,6 +52,8 @@ func InitMetrics(registry *prometheus.Registry) {
 
 	// NOTES:
 	// * Consider merging ok and errors into one with status label
+
+	registry.MustRegister(NewBPFCollector())
 }
 
 func InitMetricsForDocs(registry *prometheus.Registry) {

pkg/metrics/kprobemetrics/missed.go

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+package kprobemetrics
+
+import (
+	"sync"
+
+	"github.com/cilium/tetragon/pkg/metrics"
+	"github.com/cilium/tetragon/pkg/metrics/consts"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type MissedKey struct {
+	id     uint32
+	attach string
+}
+
+type MissedStat struct {
+	policy string
+	missed float64
+}
+
+var (
+	MissedProbes = metrics.NewBPFCounter(prometheus.NewDesc(
+		prometheus.BuildFQName(consts.MetricsNamespace, "", "missed_probes_total"),
+		"The total number of Tetragon probe missed per policy,probe.",
+		[]string{"policy", "attach"}, nil,
+	))
+
+	lock        sync.Mutex
+	missedStats = make(map[MissedKey]*MissedStat)
+)
+
+func MissedStore(id uint32, policy, attach string, missed float64) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	key := MissedKey{id, attach}
+	if stat, found := missedStats[key]; found {
+		stat.missed = missed
+	} else {
+		missedStats[key] = &MissedStat{
+			policy: policy,
+			missed: missed,
+		}
+	}
+}
+
+func MissedRemove(id uint32, attach string) {
+	lock.Lock()
+	defer lock.Unlock()
+	delete(missedStats, MissedKey{id, attach})
+}