From 517ca28b53177107a964f80482fdf66187806fde Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 24 May 2024 11:19:55 -0700 Subject: [PATCH 01/44] WIP --- .gitmodules | 3 + go.mod | 5 +- .../sessionmd/add_session_metadata.go | 9 +- .../provider/quark_provider/quark_provider.go | 150 ++++++++++++++++++ x-pack/auditbeat/processors/sessionmd/quark | 1 + 5 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 x-pack/auditbeat/processors/sessionmd/provider/quark_provider/quark_provider.go create mode 160000 x-pack/auditbeat/processors/sessionmd/quark diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..3a8bc15fbef --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "x-pack/auditbeat/processors/sessionmd/quark"] + path = x-pack/auditbeat/processors/sessionmd/quark + url = git@github.com:elastic/quark.git diff --git a/go.mod b/go.mod index 3f2e28cd2fb..b4957475cf2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.21.0 +go 1.21.1 toolchain go1.21.10 @@ -290,6 +290,7 @@ require ( github.com/elastic/elastic-transport-go/v8 v8.5.0 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/pkcs8 v1.0.0 // indirect + github.com/elastic/quark/go v0.0.0-00010101000000-000000000000 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fearful-symmetry/gomsr v0.0.1 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect @@ -419,6 +420,8 @@ replace ( github.com/dgraph-io/ristretto => github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 // Removes glog dependency. See https://github.com/elastic/beats/issues/31810. github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 + + github.com/elastic/quark/go => /home/mwolf/git/beats/x-pack/auditbeat/processors/sessionmd/quark/go github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c github.com/g8rswimmer/go-sfdc => github.com/elastic/go-sfdc v0.0.0-20201201191151-3190c381b3e1 diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 4fa86c25d02..b633cd25d6b 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfs_provider" + quarkprovider "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/quark_provider" cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -82,8 +83,14 @@ func New(cfg *cfg.C) (beat.Processor, error) { case "procfs": p, err = procfs_provider.NewProvider(ctx, logger, db, reader, c.PIDField) if err != nil { - return nil, fmt.Errorf("failed to create ebpf provider: %w", err) + return nil, fmt.Errorf("failed to create procfs provider: %w", err) + } + case "quark": + p, err = quarkprovider.NewProvider(ctx, logger, db) + if err != nil { + return nil, fmt.Errorf("failed to create quark provider: %w", err) } + default: return nil, fmt.Errorf("unknown backend configuration") } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quark_provider/quark_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/quark_provider/quark_provider.go new file mode 100644 index 00000000000..111809e3f01 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/provider/quark_provider/quark_provider.go @@ -0,0 +1,150 @@ +//go:build linux + +package quark_provider + +import ( + "context" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/elastic-agent-libs/logp" + quark "github.com/elastic/quark/go" +) + +type prvdr struct { + ctx context.Context + logger *logp.Logger + db *processdb.DB +} + +func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { + p := prvdr{ + ctx: ctx, + logger: logger, + db: db, + } + + _, err := quark.OpenQueue(127) + if err != nil { + return nil, err + } + +// w, err := ebpf.GetWatcher() +// if err != nil { +// return nil, fmt.Errorf("get ebpf watcher: %w", err) +// } +// +// records := w.Subscribe(name, eventMask) +// +// go func(logger logp.Logger) { +// for { +// r := <-records +// if r.Error != nil { +// logger.Warnw("received error from the ebpf subscription", "error", err) +// continue +// } +// if r.Event == nil { +// continue +// } +// ev := r.Event +// switch ev.Type { +// case ebpfevents.EventTypeProcessFork: +// body, ok := ev.Body.(*ebpfevents.ProcessFork) +// if !ok { +// logger.Errorf("unexpected event body, got %T", ev.Body) +// continue +// } +// pe := types.ProcessForkEvent{ +// ParentPIDs: types.PIDInfo{ +// Tid: body.ParentPids.Tid, +// Tgid: body.ParentPids.Tgid, +// Ppid: body.ParentPids.Ppid, +// Pgid: body.ParentPids.Pgid, +// Sid: body.ParentPids.Sid, +// StartTimeNS: body.ParentPids.StartTimeNs, +// }, +// ChildPIDs: types.PIDInfo{ +// Tid: body.ChildPids.Tid, +// Tgid: body.ChildPids.Tgid, +// Ppid: body.ChildPids.Ppid, +// Pgid: body.ChildPids.Pgid, +// Sid: body.ChildPids.Sid, +// StartTimeNS: body.ChildPids.StartTimeNs, +// }, +// Creds: types.CredInfo{ +// Ruid: body.Creds.Ruid, +// Rgid: body.Creds.Rgid, +// Euid: body.Creds.Euid, +// Egid: body.Creds.Egid, +// Suid: body.Creds.Suid, +// Sgid: body.Creds.Sgid, +// CapPermitted: body.Creds.CapPermitted, +// CapEffective: body.Creds.CapEffective, +// }, +// } +// p.db.InsertFork(pe) +// case ebpfevents.EventTypeProcessExec: +// body, ok := ev.Body.(*ebpfevents.ProcessExec) +// if !ok { +// logger.Errorf("unexpected event body") +// continue +// } +// pe := types.ProcessExecEvent{ +// PIDs: types.PIDInfo{ +// Tid: body.Pids.Tid, +// Tgid: body.Pids.Tgid, +// Ppid: body.Pids.Ppid, +// Pgid: body.Pids.Pgid, +// Sid: body.Pids.Sid, +// StartTimeNS: body.Pids.StartTimeNs, +// }, +// Creds: types.CredInfo{ +// Ruid: body.Creds.Ruid, +// Rgid: body.Creds.Rgid, +// Euid: body.Creds.Euid, +// Egid: body.Creds.Egid, +// Suid: body.Creds.Suid, +// Sgid: body.Creds.Sgid, +// CapPermitted: body.Creds.CapPermitted, +// CapEffective: body.Creds.CapEffective, +// }, +// CTTY: types.TTYDev{ +// Major: body.CTTY.Major, +// Minor: body.CTTY.Minor, +// }, +// CWD: body.Cwd, +// Argv: body.Argv, +// Env: body.Env, +// Filename: body.Filename, +// } +// p.db.InsertExec(pe) +// case ebpfevents.EventTypeProcessExit: +// body, ok := ev.Body.(*ebpfevents.ProcessExit) +// if !ok { +// logger.Errorf("unexpected event body") +// continue +// } +// pe := types.ProcessExitEvent{ +// PIDs: types.PIDInfo{ +// Tid: body.Pids.Tid, +// Tgid: body.Pids.Tgid, +// Ppid: body.Pids.Ppid, +// Pgid: body.Pids.Pgid, +// Sid: body.Pids.Sid, +// StartTimeNS: body.Pids.StartTimeNs, +// }, +// ExitCode: body.ExitCode, +// } +// p.db.InsertExit(pe) +// } +// } +// }(*p.logger) + + return &p, nil +} + +func (p prvdr) SyncDB(event *beat.Event, pid uint32) error { + return nil +} + diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark new file mode 160000 index 00000000000..82a6f92d5cd --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -0,0 +1 @@ +Subproject commit 82a6f92d5cdc9b53a1a1e3e7a4306c50220d9764 From fc949ded5295c929c3e35cd2cf0a25e3ba7c71bb Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 27 May 2024 15:50:03 -0700 Subject: [PATCH 02/44] rename packages --- .../sessionmd/add_session_metadata.go | 14 ++-- .../ebpfprovider.go} | 2 +- .../provider/kprobeprovider/kprobeprovider.go | 70 +++++++++++++++++++ .../procfsprovider.go} | 2 +- .../procfsprovider_test.go} | 2 +- .../quark_provider.go | 2 +- 6 files changed, 81 insertions(+), 11 deletions(-) rename x-pack/auditbeat/processors/sessionmd/provider/{ebpf_provider/ebpf_provider.go => ebpfprovider/ebpfprovider.go} (99%) create mode 100644 x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go rename x-pack/auditbeat/processors/sessionmd/provider/{procfs_provider/procfs_provider.go => procfsprovider/procfsprovider.go} (99%) rename x-pack/auditbeat/processors/sessionmd/provider/{procfs_provider/procfs_provider_test.go => procfsprovider/procfsprovider_test.go} (99%) rename x-pack/auditbeat/processors/sessionmd/provider/{quark_provider => quarkprovider}/quark_provider.go (99%) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index b633cd25d6b..a0ab909f59d 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -17,9 +17,9 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfs_provider" - quarkprovider "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/quark_provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider" cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -64,10 +64,10 @@ func New(cfg *cfg.C) (beat.Processor, error) { switch c.Backend { case "auto": - p, err = ebpf_provider.NewProvider(ctx, logger, db) + p, err = ebpfprovider.NewProvider(ctx, logger, db) if err != nil { // Most likely cause of error is not supporting ebpf on system, try procfs - p, err = procfs_provider.NewProvider(ctx, logger, db, reader, c.PIDField) + p, err = procfsprovider.NewProvider(ctx, logger, db, reader, c.PIDField) if err != nil { return nil, fmt.Errorf("failed to create provider: %w", err) } @@ -76,12 +76,12 @@ func New(cfg *cfg.C) (beat.Processor, error) { logger.Info("backend=auto using ebpf") } case "ebpf": - p, err = ebpf_provider.NewProvider(ctx, logger, db) + p, err = ebpfprovider.NewProvider(ctx, logger, db) if err != nil { return nil, fmt.Errorf("failed to create ebpf provider: %w", err) } case "procfs": - p, err = procfs_provider.NewProvider(ctx, logger, db, reader, c.PIDField) + p, err = procfsprovider.NewProvider(ctx, logger, db, reader, c.PIDField) if err != nil { return nil, fmt.Errorf("failed to create procfs provider: %w", err) } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go similarity index 99% rename from x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go rename to x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go index 8c08ae199a7..a8218e40736 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go @@ -4,7 +4,7 @@ //go:build linux -package ebpf_provider +package ebpfprovider import ( "context" diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go new file mode 100644 index 00000000000..62632a95c28 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go @@ -0,0 +1,70 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package kprobeprovider + +import ( + "bufio" + "context" + "fmt" + "os" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/elastic-agent-libs/logp" +) + +const ( + tracingEvents = "/sys/kernel/debug/tracing/kprobe_events" + tracingPipe = "/sys/kernel/tracing/trace_pipe" + + loadExecve = "p:kprobes/my_probe sys_execve\n" +) + +type prvdr struct { + ctx context.Context + logger *logp.Logger + db *processdb.DB +} + +func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { + p := prvdr{ + ctx: ctx, + logger: logger, + db: db, + } + + // Load kprobe + eventsFile, err := os.OpenFile(tracingEvents, os.O_APPEND | os.O_RDWR, 0777) + if err != nil { + return nil, fmt.Errorf("opening %v: %v", tracingEvents, err) + } + defer eventsFile.Close() + + if _, err := eventsFile.WriteString(loadExecve); err != nil { + return nil, fmt.Errorf("loading execve kprobe: %v", err) + } + + pipeFile, err := os.OpenFile(tracingPipe, os.O_RDONLY, 0644) + //Read from trace pipe, and insert events into DB. + go func(f *os.File, logger *logp.Logger) { + reader := bufio.NewReader(f) + for { + line, err := reader.ReadString('\n') + if err != nil { + logger.Errorf("reading event pipe: %v", err) + } + logger.Errorf("MWOLF: pipe: %v", line) + } + }(pipeFile, logger) + return &p, nil +} + +func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { + return nil +} + diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfs_provider/procfs_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go similarity index 99% rename from x-pack/auditbeat/processors/sessionmd/provider/procfs_provider/procfs_provider.go rename to x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index 1578e89f915..a7d5b9ed390 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfs_provider/procfs_provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -4,7 +4,7 @@ //go:build linux -package procfs_provider +package procfsprovider import ( "context" diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfs_provider/procfs_provider_test.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider_test.go similarity index 99% rename from x-pack/auditbeat/processors/sessionmd/provider/procfs_provider/procfs_provider_test.go rename to x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider_test.go index 455cb3c0433..42f19f488ce 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfs_provider/procfs_provider_test.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider_test.go @@ -4,7 +4,7 @@ //go:build linux -package procfs_provider +package procfsprovider import ( "context" diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quark_provider/quark_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quark_provider.go similarity index 99% rename from x-pack/auditbeat/processors/sessionmd/provider/quark_provider/quark_provider.go rename to x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quark_provider.go index 111809e3f01..0908b11b933 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quark_provider/quark_provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quark_provider.go @@ -1,6 +1,6 @@ //go:build linux -package quark_provider +package quarkprovider import ( "context" From 0dab06c514a0524751e835b4eeb27b8465ffd396 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 29 May 2024 23:47:14 +0000 Subject: [PATCH 03/44] Get provider working --- .../sessionmd/add_session_metadata.go | 6 +- .../processors/sessionmd/processdb/db.go | 7 + .../provider/quarkprovider/quark_provider.go | 150 ------------- .../provider/quarkprovider/quarkprovider.go | 202 ++++++++++++++++++ x-pack/auditbeat/processors/sessionmd/quark | 2 +- x-pack/auditbeat/seccomp_linux.go | 1 + 6 files changed, 214 insertions(+), 154 deletions(-) delete mode 100644 x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quark_provider.go create mode 100644 x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index a0ab909f59d..978da807c29 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -57,8 +57,9 @@ func New(cfg *cfg.C) (beat.Processor, error) { return nil, fmt.Errorf("failed to create DB: %w", err) } - backfilledPIDs := db.ScrapeProcfs() - logger.Infof("backfilled %d processes", len(backfilledPIDs)) + //TODO: backfill if not using quark +// backfilledPIDs := db.ScrapeProcfs() +// logger.Infof("backfilled %d processes", len(backfilledPIDs)) var p provider.Provider @@ -90,7 +91,6 @@ func New(cfg *cfg.C) (beat.Processor, error) { if err != nil { return nil, fmt.Errorf("failed to create quark provider: %w", err) } - default: return nil, fmt.Errorf("unknown backend configuration") } diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db.go b/x-pack/auditbeat/processors/sessionmd/processdb/db.go index 28c848ddfdb..d4e5b6876ed 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/db.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db.go @@ -254,6 +254,13 @@ func (db *DB) InsertFork(fork types.ProcessForkEvent) { } } +func (db *DB) InsertProcess(process Process) { + db.mutex.Lock() + defer db.mutex.Unlock() + + db.insertProcess(process) +} + func (db *DB) insertProcess(process Process) { pid := process.PIDs.Tgid db.processes[pid] = process diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quark_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quark_provider.go deleted file mode 100644 index 0908b11b933..00000000000 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quark_provider.go +++ /dev/null @@ -1,150 +0,0 @@ -//go:build linux - -package quarkprovider - -import ( - "context" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/elastic-agent-libs/logp" - quark "github.com/elastic/quark/go" -) - -type prvdr struct { - ctx context.Context - logger *logp.Logger - db *processdb.DB -} - -func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { - p := prvdr{ - ctx: ctx, - logger: logger, - db: db, - } - - _, err := quark.OpenQueue(127) - if err != nil { - return nil, err - } - -// w, err := ebpf.GetWatcher() -// if err != nil { -// return nil, fmt.Errorf("get ebpf watcher: %w", err) -// } -// -// records := w.Subscribe(name, eventMask) -// -// go func(logger logp.Logger) { -// for { -// r := <-records -// if r.Error != nil { -// logger.Warnw("received error from the ebpf subscription", "error", err) -// continue -// } -// if r.Event == nil { -// continue -// } -// ev := r.Event -// switch ev.Type { -// case ebpfevents.EventTypeProcessFork: -// body, ok := ev.Body.(*ebpfevents.ProcessFork) -// if !ok { -// logger.Errorf("unexpected event body, got %T", ev.Body) -// continue -// } -// pe := types.ProcessForkEvent{ -// ParentPIDs: types.PIDInfo{ -// Tid: body.ParentPids.Tid, -// Tgid: body.ParentPids.Tgid, -// Ppid: body.ParentPids.Ppid, -// Pgid: body.ParentPids.Pgid, -// Sid: body.ParentPids.Sid, -// StartTimeNS: body.ParentPids.StartTimeNs, -// }, -// ChildPIDs: types.PIDInfo{ -// Tid: body.ChildPids.Tid, -// Tgid: body.ChildPids.Tgid, -// Ppid: body.ChildPids.Ppid, -// Pgid: body.ChildPids.Pgid, -// Sid: body.ChildPids.Sid, -// StartTimeNS: body.ChildPids.StartTimeNs, -// }, -// Creds: types.CredInfo{ -// Ruid: body.Creds.Ruid, -// Rgid: body.Creds.Rgid, -// Euid: body.Creds.Euid, -// Egid: body.Creds.Egid, -// Suid: body.Creds.Suid, -// Sgid: body.Creds.Sgid, -// CapPermitted: body.Creds.CapPermitted, -// CapEffective: body.Creds.CapEffective, -// }, -// } -// p.db.InsertFork(pe) -// case ebpfevents.EventTypeProcessExec: -// body, ok := ev.Body.(*ebpfevents.ProcessExec) -// if !ok { -// logger.Errorf("unexpected event body") -// continue -// } -// pe := types.ProcessExecEvent{ -// PIDs: types.PIDInfo{ -// Tid: body.Pids.Tid, -// Tgid: body.Pids.Tgid, -// Ppid: body.Pids.Ppid, -// Pgid: body.Pids.Pgid, -// Sid: body.Pids.Sid, -// StartTimeNS: body.Pids.StartTimeNs, -// }, -// Creds: types.CredInfo{ -// Ruid: body.Creds.Ruid, -// Rgid: body.Creds.Rgid, -// Euid: body.Creds.Euid, -// Egid: body.Creds.Egid, -// Suid: body.Creds.Suid, -// Sgid: body.Creds.Sgid, -// CapPermitted: body.Creds.CapPermitted, -// CapEffective: body.Creds.CapEffective, -// }, -// CTTY: types.TTYDev{ -// Major: body.CTTY.Major, -// Minor: body.CTTY.Minor, -// }, -// CWD: body.Cwd, -// Argv: body.Argv, -// Env: body.Env, -// Filename: body.Filename, -// } -// p.db.InsertExec(pe) -// case ebpfevents.EventTypeProcessExit: -// body, ok := ev.Body.(*ebpfevents.ProcessExit) -// if !ok { -// logger.Errorf("unexpected event body") -// continue -// } -// pe := types.ProcessExitEvent{ -// PIDs: types.PIDInfo{ -// Tid: body.Pids.Tid, -// Tgid: body.Pids.Tgid, -// Ppid: body.Pids.Ppid, -// Pgid: body.Pids.Pgid, -// Sid: body.Pids.Sid, -// StartTimeNS: body.Pids.StartTimeNs, -// }, -// ExitCode: body.ExitCode, -// } -// p.db.InsertExit(pe) -// } -// } -// }(*p.logger) - - return &p, nil -} - -func (p prvdr) SyncDB(event *beat.Event, pid uint32) error { - return nil -} - diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go new file mode 100644 index 00000000000..770b742867a --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -0,0 +1,202 @@ +//go:build linux + +package quarkprovider + +import ( + "context" + "fmt" + "time" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + "github.com/elastic/elastic-agent-libs/logp" + quark "github.com/elastic/quark/go" +) + +type prvdr struct { + ctx context.Context + logger *logp.Logger + db *processdb.DB +} + +func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { + p := prvdr{ + ctx: ctx, + logger: logger, + db: db, + } + + qq, err := quark.OpenQueue(quark.QueueAttr{ + Flags: (quark.QQ_KPROBE), + MaxLength: 1000, + }, 64) + if err != nil { + return nil, fmt.Errorf("open queue: %v", err) + } + + pid1 := qq.Lookup(1) + if pid1 != nil { + logger.Error("MWOLF: got PID!") + } + + go func(qq *quark.Queue, logger *logp.Logger){ + for { + qevs, err := qq.GetEvents() + if err != nil { + logger.Errorf("get events from quark: %v", err) + continue + } + for _, qev := range qevs { + pr := processdb.Process { + PIDs: types.PIDInfo { + Tid: qev.Pid, + Tgid: qev.Pid, + Ppid: qev.Proc.Ppid, + Pgid: qev.Pid, + Sid: qev.Proc.Sid, + StartTimeNS: qev.Proc.TimeBoot, + }, + Creds: types.CredInfo{ + Ruid: qev.Proc.Uid, + Rgid: qev.Proc.Gid, + Euid: qev.Proc.Euid, + Egid: qev.Proc.Egid, + Suid: qev.Proc.Suid, + Sgid: qev.Proc.Sgid, + CapPermitted: qev.Proc.CapPermitted, + CapEffective: qev.Proc.CapEffective, + }, + Cwd: qev.Cwd, + Argv: qev.Cmdline, + Filename: qev.Comm, + } + if qev.ExitEvent != nil { + pr.ExitCode = qev.ExitEvent.ExitCode + } + logger.Errorf("MWOLF: Inserting PID %v", pr.PIDs.Tgid) + p.db.InsertProcess(pr) + +// if qev.ExitEvent == nil { +// pe := types.ProcessExecEvent{ +// PIDs: types.PIDInfo { +// Tid: qev.Pid, +// Tgid: qev.Pid, +// Ppid: qev.Proc.Ppid, +// Pgid: qev.Pid, +// Sid: qev.Proc.Sid, +// StartTimeNS: qev.Proc.TimeBoot, +// }, +// Creds: types.CredInfo{ +// Ruid: qev.Proc.Uid, +// Rgid: qev.Proc.Gid, +// Euid: qev.Proc.Euid, +// Egid: qev.Proc.Egid, +// Suid: qev.Proc.Suid, +// Sgid: qev.Proc.Sgid, +// CapPermitted: qev.Proc.CapPermitted, +// CapEffective: qev.Proc.CapEffective, +// }, +// CWD: qev.Cwd, +// Argv: qev.Cmdline, +// Filename: qev.Comm, +// } +// p.db.InsertExec(pe) +// } else { +// // Exit event +// pe := types.ProcessExitEvent { +// PIDs: types.PIDInfo { +// Tid: qev.Pid, +// Tgid: qev.Pid, +// Ppid: qev.Proc.Ppid, +// Sid: qev.Proc.Sid, +// StartTimeNS: qev.Proc.TimeBoot, +// }, +// ExitCode: qev.ExitEvent.ExitCode, +// } +// p.db.InsertExit(pe) +// } + } + if len(qevs) == 0 { + err = qq.Block() + if err != nil { + logger.Errorf("quark block: %v", err) + continue + } + } + } + }(qq, logger) + + return &p, nil +} + +const ( + maxWaitLimit = 4500 * time.Millisecond // Maximum time SyncDB will wait for process + combinedWaitLimit = 6 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration + backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time + resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset +) + +var ( + combinedWait = 0 * time.Millisecond + inBackoff = false + backoffStart = time.Now() + since = time.Now() + backoffSkipped = 0 +) + + +func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { + if s.db.HasProcess(pid) { + return nil + } + + now := time.Now() + if inBackoff { + if now.Sub(backoffStart) > backoffDuration { + s.logger.Warnf("ended backoff, skipped %d processes", backoffSkipped) + inBackoff = false + combinedWait = 0 * time.Millisecond + } else { + backoffSkipped += 1 + return nil + } + } else { + if combinedWait > combinedWaitLimit { + s.logger.Warn("starting backoff") + inBackoff = true + backoffStart = now + backoffSkipped = 0 + return nil + } + // maintain a moving window of time for the delays we track + if now.Sub(since) > resetDuration { + since = now + combinedWait = 0 * time.Millisecond + } + } + + start := now + nextWait := 5 * time.Millisecond + for { + waited := time.Since(start) + if s.db.HasProcess(pid) { + s.logger.Debugf("got process that was missing after %v", waited) + combinedWait = combinedWait + waited + return nil + } + if waited >= maxWaitLimit { + e := fmt.Errorf("process %v was not seen after %v", pid, waited) + s.logger.Warnf("%w", e) + combinedWait = combinedWait + waited + return e + } + time.Sleep(nextWait) + if nextWait*2+waited > maxWaitLimit { + nextWait = maxWaitLimit - waited + } else { + nextWait = nextWait * 2 + } + } +} diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark index 82a6f92d5cd..e238f6f9fcc 160000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -1 +1 @@ -Subproject commit 82a6f92d5cdc9b53a1a1e3e7a4306c50220d9764 +Subproject commit e238f6f9fcca7d43f05d2b8fb197e73239764f9a diff --git a/x-pack/auditbeat/seccomp_linux.go b/x-pack/auditbeat/seccomp_linux.go index 9bde50c77d0..567e7f1a83d 100644 --- a/x-pack/auditbeat/seccomp_linux.go +++ b/x-pack/auditbeat/seccomp_linux.go @@ -17,6 +17,7 @@ func init() { // requirements beyond the default policy from libbeat so whitelist // these additional syscalls. if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, + "faccessat2", "mremap", "umask", ); err != nil { From f0c76b2549ad3c8d1cc034f89ff8cd5253fbb95a Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 3 Jun 2024 11:26:36 -0700 Subject: [PATCH 04/44] Add pgid and tty info --- .../sessionmd/add_session_metadata.go | 6 +- .../provider/kprobeprovider/kprobeprovider.go | 5 +- .../provider/quarkprovider/quarkprovider.go | 129 +++++++++--------- 3 files changed, 73 insertions(+), 67 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 978da807c29..e6eefcbc26d 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -17,9 +17,9 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider" cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -58,8 +58,8 @@ func New(cfg *cfg.C) (beat.Processor, error) { } //TODO: backfill if not using quark -// backfilledPIDs := db.ScrapeProcfs() -// logger.Infof("backfilled %d processes", len(backfilledPIDs)) + // backfilledPIDs := db.ScrapeProcfs() + // logger.Infof("backfilled %d processes", len(backfilledPIDs)) var p provider.Provider diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go index 62632a95c28..bf49263c7fd 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go @@ -20,7 +20,7 @@ import ( const ( tracingEvents = "/sys/kernel/debug/tracing/kprobe_events" - tracingPipe = "/sys/kernel/tracing/trace_pipe" + tracingPipe = "/sys/kernel/tracing/trace_pipe" loadExecve = "p:kprobes/my_probe sys_execve\n" ) @@ -39,7 +39,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr } // Load kprobe - eventsFile, err := os.OpenFile(tracingEvents, os.O_APPEND | os.O_RDWR, 0777) + eventsFile, err := os.OpenFile(tracingEvents, os.O_APPEND|os.O_RDWR, 0777) if err != nil { return nil, fmt.Errorf("opening %v: %v", tracingEvents, err) } @@ -67,4 +67,3 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { return nil } - diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 770b742867a..5d77e2314b3 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + //go:build linux package quarkprovider @@ -29,7 +33,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr } qq, err := quark.OpenQueue(quark.QueueAttr{ - Flags: (quark.QQ_KPROBE), + Flags: (quark.QQ_KPROBE), MaxLength: 1000, }, 64) if err != nil { @@ -41,7 +45,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr logger.Error("MWOLF: got PID!") } - go func(qq *quark.Queue, logger *logp.Logger){ + go func(qq *quark.Queue, logger *logp.Logger) { for { qevs, err := qq.GetEvents() if err != nil { @@ -49,27 +53,31 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr continue } for _, qev := range qevs { - pr := processdb.Process { - PIDs: types.PIDInfo { - Tid: qev.Pid, - Tgid: qev.Pid, - Ppid: qev.Proc.Ppid, - Pgid: qev.Pid, - Sid: qev.Proc.Sid, + pr := processdb.Process{ + PIDs: types.PIDInfo{ + Tid: qev.Pid, + Tgid: qev.Pid, + Ppid: qev.Proc.Ppid, + Pgid: qev.Proc.Pgid, + Sid: qev.Proc.Sid, StartTimeNS: qev.Proc.TimeBoot, }, Creds: types.CredInfo{ - Ruid: qev.Proc.Uid, - Rgid: qev.Proc.Gid, - Euid: qev.Proc.Euid, - Egid: qev.Proc.Egid, - Suid: qev.Proc.Suid, - Sgid: qev.Proc.Sgid, + Ruid: qev.Proc.Uid, + Rgid: qev.Proc.Gid, + Euid: qev.Proc.Euid, + Egid: qev.Proc.Egid, + Suid: qev.Proc.Suid, + Sgid: qev.Proc.Sgid, CapPermitted: qev.Proc.CapPermitted, CapEffective: qev.Proc.CapEffective, - }, - Cwd: qev.Cwd, - Argv: qev.Cmdline, + }, + CTTY: types.TTYDev{ + Major: uint16(qev.Proc.TtyMajor), + Minor: uint16(qev.Proc.TtyMinor), + }, + Cwd: qev.Cwd, + Argv: qev.Cmdline, Filename: qev.Comm, } if qev.ExitEvent != nil { @@ -78,45 +86,45 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr logger.Errorf("MWOLF: Inserting PID %v", pr.PIDs.Tgid) p.db.InsertProcess(pr) -// if qev.ExitEvent == nil { -// pe := types.ProcessExecEvent{ -// PIDs: types.PIDInfo { -// Tid: qev.Pid, -// Tgid: qev.Pid, -// Ppid: qev.Proc.Ppid, -// Pgid: qev.Pid, -// Sid: qev.Proc.Sid, -// StartTimeNS: qev.Proc.TimeBoot, -// }, -// Creds: types.CredInfo{ -// Ruid: qev.Proc.Uid, -// Rgid: qev.Proc.Gid, -// Euid: qev.Proc.Euid, -// Egid: qev.Proc.Egid, -// Suid: qev.Proc.Suid, -// Sgid: qev.Proc.Sgid, -// CapPermitted: qev.Proc.CapPermitted, -// CapEffective: qev.Proc.CapEffective, -// }, -// CWD: qev.Cwd, -// Argv: qev.Cmdline, -// Filename: qev.Comm, -// } -// p.db.InsertExec(pe) -// } else { -// // Exit event -// pe := types.ProcessExitEvent { -// PIDs: types.PIDInfo { -// Tid: qev.Pid, -// Tgid: qev.Pid, -// Ppid: qev.Proc.Ppid, -// Sid: qev.Proc.Sid, -// StartTimeNS: qev.Proc.TimeBoot, -// }, -// ExitCode: qev.ExitEvent.ExitCode, -// } -// p.db.InsertExit(pe) -// } + // if qev.ExitEvent == nil { + // pe := types.ProcessExecEvent{ + // PIDs: types.PIDInfo { + // Tid: qev.Pid, + // Tgid: qev.Pid, + // Ppid: qev.Proc.Ppid, + // Pgid: qev.Pid, + // Sid: qev.Proc.Sid, + // StartTimeNS: qev.Proc.TimeBoot, + // }, + // Creds: types.CredInfo{ + // Ruid: qev.Proc.Uid, + // Rgid: qev.Proc.Gid, + // Euid: qev.Proc.Euid, + // Egid: qev.Proc.Egid, + // Suid: qev.Proc.Suid, + // Sgid: qev.Proc.Sgid, + // CapPermitted: qev.Proc.CapPermitted, + // CapEffective: qev.Proc.CapEffective, + // }, + // CWD: qev.Cwd, + // Argv: qev.Cmdline, + // Filename: qev.Comm, + // } + // p.db.InsertExec(pe) + // } else { + // // Exit event + // pe := types.ProcessExitEvent { + // PIDs: types.PIDInfo { + // Tid: qev.Pid, + // Tgid: qev.Pid, + // Ppid: qev.Proc.Ppid, + // Sid: qev.Proc.Sid, + // StartTimeNS: qev.Proc.TimeBoot, + // }, + // ExitCode: qev.ExitEvent.ExitCode, + // } + // p.db.InsertExit(pe) + // } } if len(qevs) == 0 { err = qq.Block() @@ -133,9 +141,9 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr const ( maxWaitLimit = 4500 * time.Millisecond // Maximum time SyncDB will wait for process - combinedWaitLimit = 6 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration - backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time - resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset + combinedWaitLimit = 6 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration + backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time + resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset ) var ( @@ -146,7 +154,6 @@ var ( backoffSkipped = 0 ) - func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { if s.db.HasProcess(pid) { return nil From 2d6054f556ac4930fbe0b6b1bb3f884fd65b3633 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 4 Jun 2024 21:33:49 +0000 Subject: [PATCH 05/44] use default attr --- .../provider/quarkprovider/quarkprovider.go | 14 ++++++++------ x-pack/auditbeat/processors/sessionmd/quark | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 5d77e2314b3..9f423807486 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -32,10 +32,9 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr db: db, } - qq, err := quark.OpenQueue(quark.QueueAttr{ - Flags: (quark.QQ_KPROBE), - MaxLength: 1000, - }, 64) + attr := quark.DefaultQueueAttr() + attr.Flags = quark.QQ_KPROBE + qq, err := quark.OpenQueue(attr, 64) if err != nil { return nil, fmt.Errorf("open queue: %v", err) } @@ -53,6 +52,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr continue } for _, qev := range qevs { + if qev.Proc == nil { + logger.Errorf("qev has no Proc: %v", qev) + continue + } pr := processdb.Process{ PIDs: types.PIDInfo{ Tid: qev.Pid, @@ -77,13 +80,12 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr Minor: uint16(qev.Proc.TtyMinor), }, Cwd: qev.Cwd, - Argv: qev.Cmdline, + Argv: []string{"foo", "bar", },//qev.Cmdline, Filename: qev.Comm, } if qev.ExitEvent != nil { pr.ExitCode = qev.ExitEvent.ExitCode } - logger.Errorf("MWOLF: Inserting PID %v", pr.PIDs.Tgid) p.db.InsertProcess(pr) // if qev.ExitEvent == nil { diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark index e238f6f9fcc..c94186cac8d 160000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -1 +1 @@ -Subproject commit e238f6f9fcca7d43f05d2b8fb197e73239764f9a +Subproject commit c94186cac8db1638694aa992f9ab3bf305d2d662 From 07b9d79664bde8d248539a850b5c3860cd8c9059 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 5 Jun 2024 18:50:23 +0000 Subject: [PATCH 06/44] WIP --- .../sessionmd/add_session_metadata.go | 28 +- .../provider/ebpfprovider/ebpfprovider.go | 4 + .../provider/kprobeprovider/kprobeprovider.go | 5 + .../provider/procfsprovider/procfsprovider.go | 4 + .../processors/sessionmd/provider/provider.go | 3 + .../provider/quarkprovider/quarkprovider.go | 331 ++++++++++-------- 6 files changed, 223 insertions(+), 152 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index e6eefcbc26d..c54d5d71844 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -20,6 +20,7 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -40,6 +41,7 @@ type addSessionMetadata struct { logger *logp.Logger db *processdb.DB provider provider.Provider + backend string } func New(cfg *cfg.C) (beat.Processor, error) { @@ -57,9 +59,10 @@ func New(cfg *cfg.C) (beat.Processor, error) { return nil, fmt.Errorf("failed to create DB: %w", err) } - //TODO: backfill if not using quark - // backfilledPIDs := db.ScrapeProcfs() - // logger.Infof("backfilled %d processes", len(backfilledPIDs)) + if c.Backend != "quark" { + backfilledPIDs := db.ScrapeProcfs() + logger.Infof("backfilled %d processes", len(backfilledPIDs)) + } var p provider.Provider @@ -99,6 +102,7 @@ func New(cfg *cfg.C) (beat.Processor, error) { logger: logger, db: db, provider: p, + backend: c.Backend, }, nil } @@ -152,13 +156,19 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { return nil, fmt.Errorf("cannot parse pid field '%s': %w", p.config.PIDField, err) } - fullProcess, err := p.db.GetProcess(pid) - if err != nil { - e := fmt.Errorf("pid %v not found in db: %w", pid, err) - p.logger.Errorf("%v", e) - return nil, e + var fullProcess types.Process + if p.backend == "quark" { + // Quark doesn't enrich with the processor DB; process info is taken directly from quark cache + i, ok := p.provider.(QuarkProvider) + fullProcess, err = p.provider.(quarkprovider).GetProcess(pid) + } else { + fullProcess, err = p.db.GetProcess(pid) + if err != nil { + e := fmt.Errorf("pid %v not found in db: %w", pid, err) + p.logger.Errorf("%v", e) + return nil, e + } } - processMap := fullProcess.ToMap() if b, err := ev.Fields.HasKey("process"); !b || err != nil { diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go index a8218e40736..7f5286e704b 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go @@ -152,6 +152,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } +func (s prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") +} + const ( maxWaitLimit = 200 * time.Millisecond // Maximum time SyncDB will wait for process combinedWaitLimit = 2 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go index bf49263c7fd..0219914437c 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" ) @@ -64,6 +65,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") +} + func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { return nil } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index a7d5b9ed390..57831fdb4d7 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -40,6 +40,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB, rea }, nil } +func (s prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") +} + // SyncDB will update the process DB with process info from procfs or the event itself func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { syscall, err := ev.GetValue(syscallField) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go index e95da3ec200..abdd0d76edd 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/provider.go @@ -8,9 +8,12 @@ package provider import ( "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + ) // SyncDB should ensure the DB is in a state to handle the event before returning. type Provider interface { SyncDB(event *beat.Event, pid uint32) error + GetProcess(pid uint32) (*types.Process, error) } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 9f423807486..dfb1f3df5eb 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -22,15 +22,10 @@ import ( type prvdr struct { ctx context.Context logger *logp.Logger - db *processdb.DB + qq *quark.Queue } func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { - p := prvdr{ - ctx: ctx, - logger: logger, - db: db, - } attr := quark.DefaultQueueAttr() attr.Flags = quark.QQ_KPROBE @@ -39,6 +34,12 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return nil, fmt.Errorf("open queue: %v", err) } + p := prvdr{ + ctx: ctx, + logger: logger, + qq: qq, + } + pid1 := qq.Lookup(1) if pid1 != nil { logger.Error("MWOLF: got PID!") @@ -51,83 +52,6 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr logger.Errorf("get events from quark: %v", err) continue } - for _, qev := range qevs { - if qev.Proc == nil { - logger.Errorf("qev has no Proc: %v", qev) - continue - } - pr := processdb.Process{ - PIDs: types.PIDInfo{ - Tid: qev.Pid, - Tgid: qev.Pid, - Ppid: qev.Proc.Ppid, - Pgid: qev.Proc.Pgid, - Sid: qev.Proc.Sid, - StartTimeNS: qev.Proc.TimeBoot, - }, - Creds: types.CredInfo{ - Ruid: qev.Proc.Uid, - Rgid: qev.Proc.Gid, - Euid: qev.Proc.Euid, - Egid: qev.Proc.Egid, - Suid: qev.Proc.Suid, - Sgid: qev.Proc.Sgid, - CapPermitted: qev.Proc.CapPermitted, - CapEffective: qev.Proc.CapEffective, - }, - CTTY: types.TTYDev{ - Major: uint16(qev.Proc.TtyMajor), - Minor: uint16(qev.Proc.TtyMinor), - }, - Cwd: qev.Cwd, - Argv: []string{"foo", "bar", },//qev.Cmdline, - Filename: qev.Comm, - } - if qev.ExitEvent != nil { - pr.ExitCode = qev.ExitEvent.ExitCode - } - p.db.InsertProcess(pr) - - // if qev.ExitEvent == nil { - // pe := types.ProcessExecEvent{ - // PIDs: types.PIDInfo { - // Tid: qev.Pid, - // Tgid: qev.Pid, - // Ppid: qev.Proc.Ppid, - // Pgid: qev.Pid, - // Sid: qev.Proc.Sid, - // StartTimeNS: qev.Proc.TimeBoot, - // }, - // Creds: types.CredInfo{ - // Ruid: qev.Proc.Uid, - // Rgid: qev.Proc.Gid, - // Euid: qev.Proc.Euid, - // Egid: qev.Proc.Egid, - // Suid: qev.Proc.Suid, - // Sgid: qev.Proc.Sgid, - // CapPermitted: qev.Proc.CapPermitted, - // CapEffective: qev.Proc.CapEffective, - // }, - // CWD: qev.Cwd, - // Argv: qev.Cmdline, - // Filename: qev.Comm, - // } - // p.db.InsertExec(pe) - // } else { - // // Exit event - // pe := types.ProcessExitEvent { - // PIDs: types.PIDInfo { - // Tid: qev.Pid, - // Tgid: qev.Pid, - // Ppid: qev.Proc.Ppid, - // Sid: qev.Proc.Sid, - // StartTimeNS: qev.Proc.TimeBoot, - // }, - // ExitCode: qev.ExitEvent.ExitCode, - // } - // p.db.InsertExit(pe) - // } - } if len(qevs) == 0 { err = qq.Block() if err != nil { @@ -141,71 +65,192 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -const ( - maxWaitLimit = 4500 * time.Millisecond // Maximum time SyncDB will wait for process - combinedWaitLimit = 6 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration - backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time - resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset -) +func (s prvdr) GetProcess(pid uint32) (*types.Process, error) { + p := s.qq.Lookup(int(pid)) + if p == nil { + return nil, fmt.Errorf("pid %v not found in quark cache", pid) + } -var ( - combinedWait = 0 * time.Millisecond - inBackoff = false - backoffStart = time.Now() - since = time.Now() - backoffSkipped = 0 -) + proc := types.Process { + EntityID: fmt.Sprintf("%s__%s", p.Comm, p.Pid), + Executable: p.Comm, + Name: p.Comm, + Start: starttime(p.Proc.TimeBoot), + End: endtime(p.ExitEvent.ExitTimeEvent), + ExitCode: p.ExitEvent.ExitCode, + Interactive: interactive(p.Proc.TtyMajor, p.Proc.TtyMinor), + WorkingDirectory: p.Cwd, + User: struct{ + ID: p.Proc.Euid, + Name: usernameFromId(p.Proc.Euid), + }, + Group: struct { + ID: p.Proc.Eguid, + Name: groupnameFromId(p.Proc.Eguid), + }, + } + return &proc, nil +} func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { - if s.db.HasProcess(pid) { - return nil - } - - now := time.Now() - if inBackoff { - if now.Sub(backoffStart) > backoffDuration { - s.logger.Warnf("ended backoff, skipped %d processes", backoffSkipped) - inBackoff = false - combinedWait = 0 * time.Millisecond - } else { - backoffSkipped += 1 - return nil - } - } else { - if combinedWait > combinedWaitLimit { - s.logger.Warn("starting backoff") - inBackoff = true - backoffStart = now - backoffSkipped = 0 - return nil - } - // maintain a moving window of time for the delays we track - if now.Sub(since) > resetDuration { - since = now - combinedWait = 0 * time.Millisecond - } - } - start := now - nextWait := 5 * time.Millisecond + timeout := 5 * time.Second + ch := time.After(timeout) for { - waited := time.Since(start) - if s.db.HasProcess(pid) { - s.logger.Debugf("got process that was missing after %v", waited) - combinedWait = combinedWait + waited - return nil - } - if waited >= maxWaitLimit { - e := fmt.Errorf("process %v was not seen after %v", pid, waited) - s.logger.Warnf("%w", e) - combinedWait = combinedWait + waited - return e - } - time.Sleep(nextWait) - if nextWait*2+waited > maxWaitLimit { - nextWait = maxWaitLimit - waited - } else { - nextWait = nextWait * 2 + select { + case <- ch: + s.logger.Errorf("%v not seen after %v", pid, timeout) + break + default: + p := s.qq.Lookup(int(pid)) + //TODO: check event, eg. make sure exit seen when enriching exit event + if p != nil { + break + } + time.Sleep(200 * time.Millisecond) } } } + +func interactiveFromTTY(tty types.TTYDev) bool { + return TTYUnknown != getTTYType(tty.Major, tty.Minor) +} + +func fullProcessFromDBProcess(p Process) types.Process { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.PIDs.StartTimeNS) + interactive := interactiveFromTTY(p.CTTY) + + ret := types.Process{ + PID: p.PIDs.Tgid, + Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Name: basename(p.Filename), + Executable: p.Filename, + Args: p.Argv, + WorkingDirectory: p.Cwd, + Interactive: &interactive, + } + + euid := p.Creds.Euid + egid := p.Creds.Egid + ret.User.ID = strconv.FormatUint(uint64(euid), 10) + username, ok := getUserName(ret.User.ID) + if ok { + ret.User.Name = username + } + ret.Group.ID = strconv.FormatUint(uint64(egid), 10) + groupname, ok := getGroupName(ret.Group.ID) + if ok { + ret.Group.Name = groupname + } + ret.Thread.Capabilities.Permitted, _ = capabilities.FromUint64(p.Creds.CapPermitted) + ret.Thread.Capabilities.Effective, _ = capabilities.FromUint64(p.Creds.CapEffective) + ret.TTY.CharDevice.Major = p.CTTY.Major + ret.TTY.CharDevice.Minor = p.CTTY.Minor + ret.ExitCode = p.ExitCode + + return ret +} + +func fillParent(process *types.Process, parent Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(parent.CTTY) + euid := parent.Creds.Euid + egid := parent.Creds.Egid + process.Parent.PID = parent.PIDs.Tgid + process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.Parent.Name = basename(parent.Filename) + process.Parent.Executable = parent.Filename + process.Parent.Args = parent.Argv + process.Parent.WorkingDirectory = parent.Cwd + process.Parent.Interactive = &interactive + process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) + username, ok := getUserName(process.Parent.User.ID) + if ok { + process.Parent.User.Name = username + } + process.Parent.Group.ID = strconv.FormatUint(uint64(egid), 10) + groupname, ok := getGroupName(process.Parent.Group.ID) + if ok { + process.Parent.Group.Name = groupname + } +} + +func fillGroupLeader(process *types.Process, groupLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(groupLeader.CTTY) + euid := groupLeader.Creds.Euid + egid := groupLeader.Creds.Egid + process.GroupLeader.PID = groupLeader.PIDs.Tgid + process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.GroupLeader.Name = basename(groupLeader.Filename) + process.GroupLeader.Executable = groupLeader.Filename + process.GroupLeader.Args = groupLeader.Argv + process.GroupLeader.WorkingDirectory = groupLeader.Cwd + process.GroupLeader.Interactive = &interactive + process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + username, ok := getUserName(process.GroupLeader.User.ID) + if ok { + process.GroupLeader.User.Name = username + } + process.GroupLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) + groupname, ok := getGroupName(process.GroupLeader.Group.ID) + if ok { + process.GroupLeader.Group.Name = groupname + } +} + +func fillSessionLeader(process *types.Process, sessionLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(sessionLeader.CTTY) + euid := sessionLeader.Creds.Euid + egid := sessionLeader.Creds.Egid + process.SessionLeader.PID = sessionLeader.PIDs.Tgid + process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.SessionLeader.Name = basename(sessionLeader.Filename) + process.SessionLeader.Executable = sessionLeader.Filename + process.SessionLeader.Args = sessionLeader.Argv + process.SessionLeader.WorkingDirectory = sessionLeader.Cwd + process.SessionLeader.Interactive = &interactive + process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + username, ok := getUserName(process.SessionLeader.User.ID) + if ok { + process.SessionLeader.User.Name = username + } + process.SessionLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) + groupname, ok := getGroupName(process.SessionLeader.Group.ID) + if ok { + process.SessionLeader.Group.Name = groupname + } +} + +func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(entryLeader.CTTY) + euid := entryLeader.Creds.Euid + egid := entryLeader.Creds.Egid + process.EntryLeader.PID = entryLeader.PIDs.Tgid + process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.EntryLeader.Name = basename(entryLeader.Filename) + process.EntryLeader.Executable = entryLeader.Filename + process.EntryLeader.Args = entryLeader.Argv + process.EntryLeader.WorkingDirectory = entryLeader.Cwd + process.EntryLeader.Interactive = &interactive + process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + username, ok := getUserName(process.EntryLeader.User.ID) + if ok { + process.EntryLeader.User.Name = username + } + process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) + groupname, ok := getGroupName(process.EntryLeader.Group.ID) + if ok { + process.EntryLeader.Group.Name = groupname + } + + process.EntryLeader.EntryMeta.Type = string(entryType) +} + + From 1f83306509586923fb2be18213525692c8e05023 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 5 Jun 2024 14:21:31 -0700 Subject: [PATCH 07/44] Enrich parent, group leader, session leader with info from quark events --- .../sessionmd/add_session_metadata.go | 3 +- .../processors/sessionmd/processdb/db.go | 6 +- .../processors/sessionmd/procfs/procfs.go | 8 +- .../provider/ebpfprovider/ebpfprovider.go | 22 +- .../provider/kprobeprovider/kprobeprovider.go | 4 +- .../provider/procfsprovider/procfsprovider.go | 18 +- .../processors/sessionmd/provider/provider.go | 2 +- .../provider/quarkprovider/quarkprovider.go | 353 +++++++++++++----- .../processors/sessionmd/types/events.go | 4 +- 9 files changed, 284 insertions(+), 136 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index c54d5d71844..55d9b1efd65 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -159,8 +159,7 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { var fullProcess types.Process if p.backend == "quark" { // Quark doesn't enrich with the processor DB; process info is taken directly from quark cache - i, ok := p.provider.(QuarkProvider) - fullProcess, err = p.provider.(quarkprovider).GetProcess(pid) + fullProcess, err = p.provider.GetProcess(pid) } else { fullProcess, err = p.db.GetProcess(pid) if err != nil { diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db.go b/x-pack/auditbeat/processors/sessionmd/processdb/db.go index d4e5b6876ed..e18c247a859 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/db.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db.go @@ -465,8 +465,8 @@ func fullProcessFromDBProcess(p Process) types.Process { } ret.Thread.Capabilities.Permitted, _ = capabilities.FromUint64(p.Creds.CapPermitted) ret.Thread.Capabilities.Effective, _ = capabilities.FromUint64(p.Creds.CapEffective) - ret.TTY.CharDevice.Major = p.CTTY.Major - ret.TTY.CharDevice.Minor = p.CTTY.Minor + ret.TTY.CharDevice.Major = uint16(p.CTTY.Major) + ret.TTY.CharDevice.Minor = uint16(p.CTTY.Minor) ret.ExitCode = p.ExitCode return ret @@ -743,7 +743,7 @@ func isFilteredExecutable(executable string) bool { return stringStartsWithEntryInList(executable, filteredExecutables[:]) } -func getTTYType(major uint16, minor uint16) TTYType { +func getTTYType(major uint32, minor uint32) TTYType { if major >= ptsMinMajor && major <= ptsMaxMajor { return Pts } diff --git a/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go index 187f9e12d8a..116d4e64901 100644 --- a/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go +++ b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go @@ -19,12 +19,12 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -func MajorTTY(ttyNr uint32) uint16 { - return uint16((ttyNr >> 8) & 0xff) +func MajorTTY(ttyNr uint32) uint32 { + return uint32((ttyNr >> 8) & 0xff) } -func MinorTTY(ttyNr uint32) uint16 { - return uint16(((ttyNr & 0xfff00000) >> 20) | (ttyNr & 0xff)) +func MinorTTY(ttyNr uint32) uint32 { + return uint32(((ttyNr >> 12) & 0xfff00) | (ttyNr & 0xff)) } // this interface exists so that we can inject a mock procfs reader for deterministic testing diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go index 7f5286e704b..e7abe29feb7 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go @@ -118,8 +118,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr CapEffective: body.Creds.CapEffective, }, CTTY: types.TTYDev{ - Major: body.CTTY.Major, - Minor: body.CTTY.Minor, + Major: uint32(body.CTTY.Major), + Minor: uint32(body.CTTY.Minor), }, CWD: body.Cwd, Argv: body.Argv, @@ -152,8 +152,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -func (s prvdr) GetProcess(pid uint32) (*types.Process, error) { - return nil, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (types.Process, error) { + return types.Process{}, fmt.Errorf("not implemented") } const ( @@ -180,15 +180,15 @@ var ( // If for some reason a lot of time has been spent waiting for missing processes, this also has a backoff timer during // which it will continue without waiting for missing events to arrive, so the processor doesn't become overly backed-up // waiting for these processes, at the cost of possibly not enriching some processes. -func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { - if s.db.HasProcess(pid) { +func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { + if p.db.HasProcess(pid) { return nil } now := time.Now() if inBackoff { if now.Sub(backoffStart) > backoffDuration { - s.logger.Warnf("ended backoff, skipped %d processes", backoffSkipped) + p.logger.Warnf("ended backoff, skipped %d processes", backoffSkipped) inBackoff = false combinedWait = 0 * time.Millisecond } else { @@ -197,7 +197,7 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { } } else { if combinedWait > combinedWaitLimit { - s.logger.Warn("starting backoff") + p.logger.Warn("starting backoff") inBackoff = true backoffStart = now backoffSkipped = 0 @@ -214,14 +214,14 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { nextWait := 5 * time.Millisecond for { waited := time.Since(start) - if s.db.HasProcess(pid) { - s.logger.Debugf("got process that was missing after %v", waited) + if p.db.HasProcess(pid) { + p.logger.Debugf("got process that was missing after %v", waited) combinedWait = combinedWait + waited return nil } if waited >= maxWaitLimit { e := fmt.Errorf("process %v was not seen after %v", pid, waited) - s.logger.Warnf("%w", e) + p.logger.Warnf("%w", e) combinedWait = combinedWait + waited return e } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go index 0219914437c..367c9018935 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go @@ -65,8 +65,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { - return nil, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (types.Process, error) { + return types.Process{}, fmt.Errorf("not implemented") } func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index 57831fdb4d7..601483e5926 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -40,12 +40,12 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB, rea }, nil } -func (s prvdr) GetProcess(pid uint32) (*types.Process, error) { - return nil, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (types.Process, error) { + return types.Process{}, fmt.Errorf("not implemented") } // SyncDB will update the process DB with process info from procfs or the event itself -func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { +func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { syscall, err := ev.GetValue(syscallField) if err != nil { return fmt.Errorf("event not supported, no syscall data") @@ -54,7 +54,7 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { switch syscall { case "execveat", "execve": pe := types.ProcessExecEvent{} - proc_info, err := s.reader.GetProcess(pid) + proc_info, err := p.reader.GetProcess(pid) if err == nil { pe.PIDs = proc_info.PIDs pe.Creds = proc_info.Creds @@ -64,7 +64,7 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { pe.Env = proc_info.Env pe.Filename = proc_info.Filename } else { - s.logger.Warnf("couldn't get process info from proc for pid %v: %w", pid, err) + p.logger.Warnf("couldn't get process info from proc for pid %v: %w", pid, err) // If process info couldn't be taken from procfs, populate with as much info as // possible from the event pe.PIDs.Tgid = pid @@ -81,7 +81,7 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { } pe.PIDs.Ppid = uint32(i) - parent, err = s.db.GetProcess(pe.PIDs.Ppid) + parent, err = p.db.GetProcess(pe.PIDs.Ppid) if err != nil { goto out } @@ -94,7 +94,7 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { pe.CWD = intr.(string) out: } - s.db.InsertExec(pe) + p.db.InsertExec(pe) if err != nil { return fmt.Errorf("insert exec to db: %w", err) } @@ -104,7 +104,7 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { Tgid: pid, }, } - s.db.InsertExit(pe) + p.db.InsertExit(pe) case "setsid": intr, err := ev.Fields.GetValue("auditd.result") if err != nil { @@ -121,7 +121,7 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { Sid: pid, }, } - s.db.InsertSetsid(setsid_ev) + p.db.InsertSetsid(setsid_ev) } } return nil diff --git a/x-pack/auditbeat/processors/sessionmd/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go index abdd0d76edd..7824f7cb45f 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/provider.go @@ -15,5 +15,5 @@ import ( // SyncDB should ensure the DB is in a state to handle the event before returning. type Provider interface { SyncDB(event *beat.Event, pid uint32) error - GetProcess(pid uint32) (*types.Process, error) + GetProcess(pid uint32) (types.Process, error) } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index dfb1f3df5eb..f955d9be38e 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -10,11 +10,19 @@ import ( "context" "fmt" "time" + "os" + "sync" + "strings" + "encoding/base64" + "strconv" + "path" + "os/user" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" "github.com/elastic/elastic-agent-libs/logp" quark "github.com/elastic/quark/go" ) @@ -25,6 +33,85 @@ type prvdr struct { qq *quark.Queue } +type TTYType int + +const ( + TTYUnknown TTYType = iota + Pts + TTY + TTYConsole +) + +type EntryType string + +const ( + Init EntryType = "init" + Sshd EntryType = "sshd" + Ssm EntryType = "ssm" + Container EntryType = "container" + Terminal EntryType = "terminal" + EntryConsole EntryType = "console" + EntryUnknown EntryType = "unknown" +) + +var containerRuntimes = [...]string{ + "containerd-shim", + "runc", + "conmon", +} + +// "filtered" executables are executables that relate to internal +// implementation details of entry mechanisms. The set of circumstances under +// which they can become an entry leader are reduced compared to other binaries +// (see implementation and unit tests). +var filteredExecutables = [...]string{ + "runc", + "containerd-shim", + "calico-node", + "check-status", + "conmon", +} + +const ( + ptsMinMajor = 136 + ptsMaxMajor = 143 + ttyMajor = 4 + consoleMaxMinor = 63 + ttyMaxMinor = 255 + retryCount = 2 +) + +var ( + bootID string + pidNsInode uint64 + initError error + once sync.Once +) + +func readBootID() (string, error) { + bootID, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + panic(fmt.Sprintf("could not read /proc/sys/kernel/random/boot_id: %v", err)) + } + + return strings.TrimRight(string(bootID), "\n"), nil +} + +func readPIDNsInode() (uint64, error) { + var ret uint64 + + pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") + if err != nil { + panic(fmt.Sprintf("could not read /proc/self/ns/pid: %v", err)) + } + + if _, err = fmt.Sscanf(pidNsInodeRaw, "pid:[%d]", &ret); err != nil { + panic(fmt.Sprintf("could not parse contents of /proc/self/ns/pid (%s): %v", pidNsInodeRaw, err)) + } + + return ret, nil +} + func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { attr := quark.DefaultQueueAttr() @@ -65,46 +152,19 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -func (s prvdr) GetProcess(pid uint32) (*types.Process, error) { - p := s.qq.Lookup(int(pid)) - if p == nil { - return nil, fmt.Errorf("pid %v not found in quark cache", pid) - } - - proc := types.Process { - EntityID: fmt.Sprintf("%s__%s", p.Comm, p.Pid), - Executable: p.Comm, - Name: p.Comm, - Start: starttime(p.Proc.TimeBoot), - End: endtime(p.ExitEvent.ExitTimeEvent), - ExitCode: p.ExitEvent.ExitCode, - Interactive: interactive(p.Proc.TtyMajor, p.Proc.TtyMinor), - WorkingDirectory: p.Cwd, - User: struct{ - ID: p.Proc.Euid, - Name: usernameFromId(p.Proc.Euid), - }, - Group: struct { - ID: p.Proc.Eguid, - Name: groupnameFromId(p.Proc.Eguid), - }, - } - return &proc, nil -} - -func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { +func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { timeout := 5 * time.Second ch := time.After(timeout) for { select { case <- ch: - s.logger.Errorf("%v not seen after %v", pid, timeout) + p.logger.Errorf("%v not seen after %v", pid, timeout) break default: - p := s.qq.Lookup(int(pid)) + proc := p.qq.Lookup(int(pid)) //TODO: check event, eg. make sure exit seen when enriching exit event - if p != nil { + if proc != nil { break } time.Sleep(200 * time.Millisecond) @@ -112,26 +172,30 @@ func (s prvdr) SyncDB(ev *beat.Event, pid uint32) error { } } -func interactiveFromTTY(tty types.TTYDev) bool { - return TTYUnknown != getTTYType(tty.Major, tty.Minor) -} +func (p prvdr) GetProcess(pid uint32) (types.Process, error) { + qev := p.qq.Lookup(int(pid)) + if qev == nil { + return types.Process{}, fmt.Errorf("PID %d not found in cache", pid) + } -func fullProcessFromDBProcess(p Process) types.Process { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.PIDs.StartTimeNS) - interactive := interactiveFromTTY(p.CTTY) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + interactive := interactiveFromTTY(types.TTYDev{ + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) ret := types.Process{ - PID: p.PIDs.Tgid, + PID: qev.Pid, Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), - Name: basename(p.Filename), - Executable: p.Filename, - Args: p.Argv, - WorkingDirectory: p.Cwd, + Name: basename(qev.Filename), + Executable: qev.Filename, +// Args: qev.Argv, + WorkingDirectory: qev.Cwd, Interactive: &interactive, } - euid := p.Creds.Euid - egid := p.Creds.Egid + euid := qev.Proc.Euid + egid := qev.Proc.Egid ret.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(ret.User.ID) if ok { @@ -142,27 +206,37 @@ func fullProcessFromDBProcess(p Process) types.Process { if ok { ret.Group.Name = groupname } - ret.Thread.Capabilities.Permitted, _ = capabilities.FromUint64(p.Creds.CapPermitted) - ret.Thread.Capabilities.Effective, _ = capabilities.FromUint64(p.Creds.CapEffective) - ret.TTY.CharDevice.Major = p.CTTY.Major - ret.TTY.CharDevice.Minor = p.CTTY.Minor - ret.ExitCode = p.ExitCode + ret.TTY.CharDevice.Major = uint16(qev.Proc.TtyMajor) + ret.TTY.CharDevice.Minor = uint16(qev.Proc.TtyMinor) + if qev.ExitEvent != nil { + ret.ExitCode = qev.ExitEvent.ExitCode + } - return ret + p.fillParent(&ret, qev.Proc.Ppid) + p.fillGroupLeader(&ret, qev.Proc.Pgid) + p.fillSessionLeader(&ret, qev.Proc.Sid) + return ret, nil } -func fillParent(process *types.Process, parent Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.PIDs.StartTimeNS) +func (p prvdr) fillParent(process *types.Process, ppid uint32) { + qev := p.qq.Lookup(int(ppid)) + if qev == nil { + return + } - interactive := interactiveFromTTY(parent.CTTY) - euid := parent.Creds.Euid - egid := parent.Creds.Egid - process.Parent.PID = parent.PIDs.Tgid + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + interactive := interactiveFromTTY(types.TTYDev{ + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) + euid := qev.Proc.Euid + egid := qev.Proc.Egid + process.Parent.PID = qev.Proc.Ppid process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.Parent.Name = basename(parent.Filename) - process.Parent.Executable = parent.Filename - process.Parent.Args = parent.Argv - process.Parent.WorkingDirectory = parent.Cwd + process.Parent.Name = basename(qev.Filename) + process.Parent.Executable = qev.Filename +// process.Parent.Args = qev.Argv + process.Parent.WorkingDirectory = qev.Cwd process.Parent.Interactive = &interactive process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(process.Parent.User.ID) @@ -176,18 +250,26 @@ func fillParent(process *types.Process, parent Process) { } } -func fillGroupLeader(process *types.Process, groupLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.PIDs.StartTimeNS) +func (p prvdr)fillGroupLeader(process *types.Process, pgid uint32) { + qev := p.qq.Lookup(int(pgid)) + if qev == nil { + return + } - interactive := interactiveFromTTY(groupLeader.CTTY) - euid := groupLeader.Creds.Euid - egid := groupLeader.Creds.Egid - process.GroupLeader.PID = groupLeader.PIDs.Tgid + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + + interactive := interactiveFromTTY(types.TTYDev{ + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) + euid := qev.Proc.Euid + egid := qev.Proc.Egid + process.GroupLeader.PID = qev.Pid process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.GroupLeader.Name = basename(groupLeader.Filename) - process.GroupLeader.Executable = groupLeader.Filename - process.GroupLeader.Args = groupLeader.Argv - process.GroupLeader.WorkingDirectory = groupLeader.Cwd + process.GroupLeader.Name = basename(qev.Filename) + process.GroupLeader.Executable = qev.Filename +// process.GroupLeader.Args = qev.Argv + process.GroupLeader.WorkingDirectory = qev.Cwd process.GroupLeader.Interactive = &interactive process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(process.GroupLeader.User.ID) @@ -201,18 +283,26 @@ func fillGroupLeader(process *types.Process, groupLeader Process) { } } -func fillSessionLeader(process *types.Process, sessionLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.PIDs.StartTimeNS) +func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { + qev := p.qq.Lookup(int(sid)) + if qev == nil { + return + } + + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) - interactive := interactiveFromTTY(sessionLeader.CTTY) - euid := sessionLeader.Creds.Euid - egid := sessionLeader.Creds.Egid - process.SessionLeader.PID = sessionLeader.PIDs.Tgid + interactive := interactiveFromTTY(types.TTYDev{ + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) + euid := qev.Proc.Euid + egid := qev.Proc.Egid + process.SessionLeader.PID = qev.Pid process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.SessionLeader.Name = basename(sessionLeader.Filename) - process.SessionLeader.Executable = sessionLeader.Filename - process.SessionLeader.Args = sessionLeader.Argv - process.SessionLeader.WorkingDirectory = sessionLeader.Cwd + process.SessionLeader.Name = basename(qev.Filename) + process.SessionLeader.Executable = qev.Filename +// process.SessionLeader.Args = qev.Argv + process.SessionLeader.WorkingDirectory = qev.Cwd process.SessionLeader.Interactive = &interactive process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(process.SessionLeader.User.ID) @@ -226,31 +316,90 @@ func fillSessionLeader(process *types.Process, sessionLeader Process) { } } -func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.PIDs.StartTimeNS) - - interactive := interactiveFromTTY(entryLeader.CTTY) - euid := entryLeader.Creds.Euid - egid := entryLeader.Creds.Egid - process.EntryLeader.PID = entryLeader.PIDs.Tgid - process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.EntryLeader.Name = basename(entryLeader.Filename) - process.EntryLeader.Executable = entryLeader.Filename - process.EntryLeader.Args = entryLeader.Argv - process.EntryLeader.WorkingDirectory = entryLeader.Cwd - process.EntryLeader.Interactive = &interactive - process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) - username, ok := getUserName(process.EntryLeader.User.ID) - if ok { - process.EntryLeader.User.Name = username +//func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { +// reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.PIDs.StartTimeNS) +// +// interactive := interactiveFromTTY(entryLeader.CTTY) +// euid := entryLeader.Creds.Euid +// egid := entryLeader.Creds.Egid +// process.EntryLeader.PID = entryLeader.PIDs.Tgid +// process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) +// process.EntryLeader.Name = basename(entryLeader.Filename) +// process.EntryLeader.Executable = entryLeader.Filename +// process.EntryLeader.Args = entryLeader.Argv +// process.EntryLeader.WorkingDirectory = entryLeader.Cwd +// process.EntryLeader.Interactive = &interactive +// process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) +// username, ok := getUserName(process.EntryLeader.User.ID) +// if ok { +// process.EntryLeader.User.Name = username +// } +// process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) +// groupname, ok := getGroupName(process.EntryLeader.Group.ID) +// if ok { +// process.EntryLeader.Group.Name = groupname +// } +// +// process.EntryLeader.EntryMeta.Type = string(entryType) +//} + +func interactiveFromTTY(tty types.TTYDev) bool { + return TTYUnknown != getTTYType(tty.Major, tty.Minor) +} + +func getTTYType(major uint32, minor uint32) TTYType { + if major >= ptsMinMajor && major <= ptsMaxMajor { + return Pts } - process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) - groupname, ok := getGroupName(process.EntryLeader.Group.ID) - if ok { - process.EntryLeader.Group.Name = groupname + + if ttyMajor == major { + if minor <= consoleMaxMinor { + return TTYConsole + } else if minor > consoleMaxMinor && minor <= ttyMaxMinor { + return TTY + } + } + + return TTYUnknown +} + +func calculateEntityIDv1(pid uint32, startTime time.Time) string { + return base64.StdEncoding.EncodeToString( + []byte( + fmt.Sprintf("%d__%s__%d__%d", + pidNsInode, + bootID, + uint64(pid), + uint64(startTime.Unix()), + ), + ), + ) +} + +// `path.Base` returns a '.' for empty strings, this just special cases that +// situation to return an empty string +func basename(pathStr string) string { + if pathStr == "" { + return "" } - process.EntryLeader.EntryMeta.Type = string(entryType) + return path.Base(pathStr) } +// getUserName will return the name associated with the user ID, if it exists +func getUserName(id string) (string, bool) { + user, err := user.LookupId(id) + if err != nil { + return "", false + } + return user.Username, true +} +// getGroupName will return the name associated with the group ID, if it exists +func getGroupName(id string) (string, bool) { + group, err := user.LookupGroupId(id) + if err != nil { + return "", false + } + return group.Name, true +} diff --git a/x-pack/auditbeat/processors/sessionmd/types/events.go b/x-pack/auditbeat/processors/sessionmd/types/events.go index 5f8d67d763f..857ab8fa2c1 100644 --- a/x-pack/auditbeat/processors/sessionmd/types/events.go +++ b/x-pack/auditbeat/processors/sessionmd/types/events.go @@ -60,8 +60,8 @@ type TTYTermios struct { } type TTYDev struct { - Minor uint16 - Major uint16 + Minor uint32 + Major uint32 Winsize TTYWinsize Termios TTYTermios } From c630b756b47edc7e2e7d04a7a34a19b9635f2d33 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 5 Jun 2024 23:01:50 +0000 Subject: [PATCH 08/44] WIP --- .../sessionmd/add_session_metadata.go | 9 +++- .../provider/ebpfprovider/ebpfprovider.go | 4 +- .../provider/kprobeprovider/kprobeprovider.go | 4 +- .../provider/procfsprovider/procfsprovider.go | 4 +- .../processors/sessionmd/provider/provider.go | 2 +- .../provider/quarkprovider/quarkprovider.go | 43 ++++++++++--------- x-pack/auditbeat/processors/sessionmd/quark | 2 +- 7 files changed, 39 insertions(+), 29 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 55d9b1efd65..8da7a105571 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -94,6 +94,7 @@ func New(cfg *cfg.C) (beat.Processor, error) { if err != nil { return nil, fmt.Errorf("failed to create quark provider: %w", err) } + db.Close() // db not used with quark default: return nil, fmt.Errorf("unknown backend configuration") } @@ -159,7 +160,13 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { var fullProcess types.Process if p.backend == "quark" { // Quark doesn't enrich with the processor DB; process info is taken directly from quark cache - fullProcess, err = p.provider.GetProcess(pid) + proc, err := p.provider.GetProcess(pid) + if err != nil { + e := fmt.Errorf("pid %v not found in db: %w", pid, err) + p.logger.Errorf("%v", e) + return nil, e + } + fullProcess = *proc } else { fullProcess, err = p.db.GetProcess(pid) if err != nil { diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go index e7abe29feb7..8b780bc16f5 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go @@ -152,8 +152,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { - return types.Process{}, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") } const ( diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go index 367c9018935..0219914437c 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go @@ -65,8 +65,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { - return types.Process{}, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") } func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index 601483e5926..af1b481c4e2 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -40,8 +40,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB, rea }, nil } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { - return types.Process{}, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") } // SyncDB will update the process DB with process info from procfs or the event itself diff --git a/x-pack/auditbeat/processors/sessionmd/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go index 7824f7cb45f..abdd0d76edd 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/provider.go @@ -15,5 +15,5 @@ import ( // SyncDB should ensure the DB is in a state to handle the event before returning. type Provider interface { SyncDB(event *beat.Event, pid uint32) error - GetProcess(pid uint32) (types.Process, error) + GetProcess(pid uint32) (*types.Process, error) } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index f955d9be38e..7ef30787cb8 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -153,29 +153,32 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr } func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - - timeout := 5 * time.Second - ch := time.After(timeout) - for { - select { - case <- ch: - p.logger.Errorf("%v not seen after %v", pid, timeout) - break - default: - proc := p.qq.Lookup(int(pid)) - //TODO: check event, eg. make sure exit seen when enriching exit event - if proc != nil { - break - } - time.Sleep(200 * time.Millisecond) - } - } + time.Sleep(500 * time.Millisecond) + return nil + +//// TODO: Not working correctly... +// timeout := 5 * time.Second +// ch := time.After(timeout) +// for { +// select { +// case <- ch: +// p.logger.Errorf("%v not seen after %v", pid, timeout) +// break +// default: +// proc := p.qq.Lookup(int(pid)) +// //TODO: check event, eg. make sure exit seen when enriching exit event +// if proc != nil { +// break +// } +// time.Sleep(200 * time.Millisecond) +// } +// } } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { qev := p.qq.Lookup(int(pid)) if qev == nil { - return types.Process{}, fmt.Errorf("PID %d not found in cache", pid) + return nil, fmt.Errorf("PID %d not found in cache", pid) } reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) @@ -215,7 +218,7 @@ func (p prvdr) GetProcess(pid uint32) (types.Process, error) { p.fillParent(&ret, qev.Proc.Ppid) p.fillGroupLeader(&ret, qev.Proc.Pgid) p.fillSessionLeader(&ret, qev.Proc.Sid) - return ret, nil + return &ret, nil } func (p prvdr) fillParent(process *types.Process, ppid uint32) { diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark index c94186cac8d..d608a9efdf2 160000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -1 +1 @@ -Subproject commit c94186cac8db1638694aa992f9ab3bf305d2d662 +Subproject commit d608a9efdf2bbd4dfaa693ba9f462ccb9491b691 From 0f701dc6dbb65ee1a43dc5ac75ef8e88338a1207 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 5 Jun 2024 23:01:50 +0000 Subject: [PATCH 09/44] WIP --- .../sessionmd/add_session_metadata.go | 11 +++- .../provider/ebpfprovider/ebpfprovider.go | 4 +- .../provider/kprobeprovider/kprobeprovider.go | 4 +- .../provider/procfsprovider/procfsprovider.go | 4 +- .../processors/sessionmd/provider/provider.go | 2 +- .../provider/quarkprovider/quarkprovider.go | 56 +++++++++---------- x-pack/auditbeat/processors/sessionmd/quark | 2 +- 7 files changed, 45 insertions(+), 38 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 55d9b1efd65..7c73a2520d9 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -90,10 +90,11 @@ func New(cfg *cfg.C) (beat.Processor, error) { return nil, fmt.Errorf("failed to create procfs provider: %w", err) } case "quark": - p, err = quarkprovider.NewProvider(ctx, logger, db) + p, err = quarkprovider.NewProvider(ctx, logger) if err != nil { return nil, fmt.Errorf("failed to create quark provider: %w", err) } + db.Close() // db not used with quark default: return nil, fmt.Errorf("unknown backend configuration") } @@ -159,7 +160,13 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { var fullProcess types.Process if p.backend == "quark" { // Quark doesn't enrich with the processor DB; process info is taken directly from quark cache - fullProcess, err = p.provider.GetProcess(pid) + proc, err := p.provider.GetProcess(pid) + if err != nil { + e := fmt.Errorf("pid %v not found in db: %w", pid, err) + p.logger.Errorf("%v", e) + return nil, e + } + fullProcess = *proc } else { fullProcess, err = p.db.GetProcess(pid) if err != nil { diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go index e7abe29feb7..8b780bc16f5 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go @@ -152,8 +152,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { - return types.Process{}, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") } const ( diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go index 367c9018935..0219914437c 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go @@ -65,8 +65,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return &p, nil } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { - return types.Process{}, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") } func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index 601483e5926..af1b481c4e2 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -40,8 +40,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB, rea }, nil } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { - return types.Process{}, fmt.Errorf("not implemented") +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("not implemented") } // SyncDB will update the process DB with process info from procfs or the event itself diff --git a/x-pack/auditbeat/processors/sessionmd/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go index 7824f7cb45f..abdd0d76edd 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/provider.go @@ -15,5 +15,5 @@ import ( // SyncDB should ensure the DB is in a state to handle the event before returning. type Provider interface { SyncDB(event *beat.Event, pid uint32) error - GetProcess(pid uint32) (types.Process, error) + GetProcess(pid uint32) (*types.Process, error) } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index f955d9be38e..401b42a9d58 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -19,7 +19,6 @@ import ( "os/user" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" @@ -112,10 +111,10 @@ func readPIDNsInode() (uint64, error) { return ret, nil } -func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { +func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, error) { attr := quark.DefaultQueueAttr() - attr.Flags = quark.QQ_KPROBE + attr.Flags = quark.QQ_KPROBE | quark.QQ_MIN_AGG qq, err := quark.OpenQueue(attr, 64) if err != nil { return nil, fmt.Errorf("open queue: %v", err) @@ -127,11 +126,6 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr qq: qq, } - pid1 := qq.Lookup(1) - if pid1 != nil { - logger.Error("MWOLF: got PID!") - } - go func(qq *quark.Queue, logger *logp.Logger) { for { qevs, err := qq.GetEvents() @@ -139,6 +133,9 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr logger.Errorf("get events from quark: %v", err) continue } + for _, qev := range qevs { + logger.Debugf("qev: %v", qev) + } if len(qevs) == 0 { err = qq.Block() if err != nil { @@ -153,29 +150,32 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr } func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - - timeout := 5 * time.Second - ch := time.After(timeout) - for { - select { - case <- ch: - p.logger.Errorf("%v not seen after %v", pid, timeout) - break - default: - proc := p.qq.Lookup(int(pid)) - //TODO: check event, eg. make sure exit seen when enriching exit event - if proc != nil { - break - } - time.Sleep(200 * time.Millisecond) - } - } + time.Sleep(500 * time.Millisecond) + return nil + +//// TODO: Not working correctly... +// timeout := 5 * time.Second +// ch := time.After(timeout) +// for { +// select { +// case <- ch: +// p.logger.Errorf("%v not seen after %v", pid, timeout) +// break +// default: +// proc := p.qq.Lookup(int(pid)) +// //TODO: check event, eg. make sure exit seen when enriching exit event +// if proc != nil { +// break +// } +// time.Sleep(200 * time.Millisecond) +// } +// } } -func (p prvdr) GetProcess(pid uint32) (types.Process, error) { +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { qev := p.qq.Lookup(int(pid)) if qev == nil { - return types.Process{}, fmt.Errorf("PID %d not found in cache", pid) + return nil, fmt.Errorf("PID %d not found in cache", pid) } reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) @@ -215,7 +215,7 @@ func (p prvdr) GetProcess(pid uint32) (types.Process, error) { p.fillParent(&ret, qev.Proc.Ppid) p.fillGroupLeader(&ret, qev.Proc.Pgid) p.fillSessionLeader(&ret, qev.Proc.Sid) - return ret, nil + return &ret, nil } func (p prvdr) fillParent(process *types.Process, ppid uint32) { diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark index c94186cac8d..d608a9efdf2 160000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -1 +1 @@ -Subproject commit c94186cac8db1638694aa992f9ab3bf305d2d662 +Subproject commit d608a9efdf2bbd4dfaa693ba9f462ccb9491b691 From c2fa8154a6ed8ca14c9ae2f6a25a80d1feae471c Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 7 Jun 2024 22:56:56 +0000 Subject: [PATCH 10/44] WIP --- .../processors/sessionmd/provider/provider.go | 1 - .../provider/quarkprovider/quarkprovider.go | 206 ++++++++++-------- 2 files changed, 111 insertions(+), 96 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go index abdd0d76edd..4ac9530cfea 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/provider.go @@ -9,7 +9,6 @@ package provider import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" - ) // SyncDB should ensure the DB is in a state to handle the event before returning. diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 401b42a9d58..bd7f94d360b 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -8,20 +8,20 @@ package quarkprovider import ( "context" + "encoding/base64" "fmt" + "os" + "os/user" + "path" + "strconv" + "strings" + "sync" "time" - "os" - "sync" - "strings" - "encoding/base64" - "strconv" - "path" - "os/user" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" "github.com/elastic/elastic-agent-libs/logp" quark "github.com/elastic/quark/go" ) @@ -150,46 +150,47 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e } func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - time.Sleep(500 * time.Millisecond) + time.Sleep(200 * time.Millisecond) return nil -//// TODO: Not working correctly... -// timeout := 5 * time.Second -// ch := time.After(timeout) -// for { -// select { -// case <- ch: -// p.logger.Errorf("%v not seen after %v", pid, timeout) -// break -// default: -// proc := p.qq.Lookup(int(pid)) -// //TODO: check event, eg. make sure exit seen when enriching exit event -// if proc != nil { -// break -// } -// time.Sleep(200 * time.Millisecond) -// } -// } + // // TODO: Not working correctly... + // + // timeout := 5 * time.Second + // ch := time.After(timeout) + // for { + // select { + // case <- ch: + // p.logger.Errorf("%v not seen after %v", pid, timeout) + // break + // default: + // proc := p.qq.Lookup(int(pid)) + // //TODO: check event, eg. make sure exit seen when enriching exit event + // if proc != nil { + // break + // } + // time.Sleep(200 * time.Millisecond) + // } + // } } func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { - qev := p.qq.Lookup(int(pid)) - if qev == nil { - return nil, fmt.Errorf("PID %d not found in cache", pid) - } + qev := p.qq.Lookup(int(pid)) + if qev == nil { + return nil, fmt.Errorf("PID %d not found in cache", pid) + } reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) ret := types.Process{ - PID: qev.Pid, - Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), - Name: basename(qev.Filename), - Executable: qev.Filename, -// Args: qev.Argv, + PID: qev.Pid, + Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Name: basename(qev.Filename), + Executable: qev.Filename, + // Args: qev.Argv, WorkingDirectory: qev.Cwd, Interactive: &interactive, } @@ -208,34 +209,36 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { } ret.TTY.CharDevice.Major = uint16(qev.Proc.TtyMajor) ret.TTY.CharDevice.Minor = uint16(qev.Proc.TtyMinor) - if qev.ExitEvent != nil { - ret.ExitCode = qev.ExitEvent.ExitCode - } + if qev.ExitEvent != nil { + ret.ExitCode = qev.ExitEvent.ExitCode + } + ret.EntityID = calculateEntityIDv1(pid, *ret.Start) - p.fillParent(&ret, qev.Proc.Ppid) - p.fillGroupLeader(&ret, qev.Proc.Pgid) - p.fillSessionLeader(&ret, qev.Proc.Sid) + p.fillParent(&ret, qev.Proc.Ppid) + p.fillGroupLeader(&ret, qev.Proc.Pgid) + p.fillSessionLeader(&ret, qev.Proc.Sid) + p.fillEntryLeader(&ret, Init, uint32(1)) return &ret, nil } func (p prvdr) fillParent(process *types.Process, ppid uint32) { - qev := p.qq.Lookup(int(ppid)) - if qev == nil { - return - } + qev := p.qq.Lookup(int(ppid)) + if qev == nil { + return + } reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) euid := qev.Proc.Euid egid := qev.Proc.Egid process.Parent.PID = qev.Proc.Ppid process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.Parent.Name = basename(qev.Filename) process.Parent.Executable = qev.Filename -// process.Parent.Args = qev.Argv + // process.Parent.Args = qev.Argv process.Parent.WorkingDirectory = qev.Cwd process.Parent.Interactive = &interactive process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -248,27 +251,28 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { if ok { process.Parent.Group.Name = groupname } + process.Parent.EntityID = calculateEntityIDv1(ppid, *process.Start) } -func (p prvdr)fillGroupLeader(process *types.Process, pgid uint32) { +func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { qev := p.qq.Lookup(int(pgid)) - if qev == nil { - return - } + if qev == nil { + return + } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) euid := qev.Proc.Euid egid := qev.Proc.Egid process.GroupLeader.PID = qev.Pid process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.GroupLeader.Name = basename(qev.Filename) process.GroupLeader.Executable = qev.Filename -// process.GroupLeader.Args = qev.Argv + // process.GroupLeader.Args = qev.Argv process.GroupLeader.WorkingDirectory = qev.Cwd process.GroupLeader.Interactive = &interactive process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -281,27 +285,28 @@ func (p prvdr)fillGroupLeader(process *types.Process, pgid uint32) { if ok { process.GroupLeader.Group.Name = groupname } + process.GroupLeader.EntityID = calculateEntityIDv1(pgid, *process.GroupLeader.Start) } func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { qev := p.qq.Lookup(int(sid)) - if qev == nil { - return - } + if qev == nil { + return + } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) euid := qev.Proc.Euid egid := qev.Proc.Egid process.SessionLeader.PID = qev.Pid process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.SessionLeader.Name = basename(qev.Filename) process.SessionLeader.Executable = qev.Filename -// process.SessionLeader.Args = qev.Argv + // process.SessionLeader.Args = qev.Argv process.SessionLeader.WorkingDirectory = qev.Cwd process.SessionLeader.Interactive = &interactive process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -314,34 +319,45 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { if ok { process.SessionLeader.Group.Name = groupname } + process.SessionLeader.EntityID = calculateEntityIDv1(sid, *process.SessionLeader.Start) } -//func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { -// reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.PIDs.StartTimeNS) -// -// interactive := interactiveFromTTY(entryLeader.CTTY) -// euid := entryLeader.Creds.Euid -// egid := entryLeader.Creds.Egid -// process.EntryLeader.PID = entryLeader.PIDs.Tgid -// process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) -// process.EntryLeader.Name = basename(entryLeader.Filename) -// process.EntryLeader.Executable = entryLeader.Filename -// process.EntryLeader.Args = entryLeader.Argv -// process.EntryLeader.WorkingDirectory = entryLeader.Cwd -// process.EntryLeader.Interactive = &interactive -// process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) -// username, ok := getUserName(process.EntryLeader.User.ID) -// if ok { -// process.EntryLeader.User.Name = username -// } -// process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) -// groupname, ok := getGroupName(process.EntryLeader.Group.ID) -// if ok { -// process.EntryLeader.Group.Name = groupname -// } -// -// process.EntryLeader.EntryMeta.Type = string(entryType) -//} +func (p prvdr) fillEntryLeader(process *types.Process, entryType EntryType, elid uint32) { + qev := p.qq.Lookup(int(elid)) + if qev == nil { + return + } + + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + + interactive := interactiveFromTTY(types.TTYDev{ + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) + + euid := qev.Proc.Euid + egid := qev.Proc.Egid + process.EntryLeader.PID = qev.Pid + process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.EntryLeader.Name = basename(qev.Filename) + process.EntryLeader.Executable = qev.Filename + // process.EntryLeader.Args = qev.Argv + process.EntryLeader.WorkingDirectory = qev.Cwd + process.EntryLeader.Interactive = &interactive + process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + username, ok := getUserName(process.EntryLeader.User.ID) + if ok { + process.EntryLeader.User.Name = username + } + process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) + groupname, ok := getGroupName(process.EntryLeader.Group.ID) + if ok { + process.EntryLeader.Group.Name = groupname + } + + process.EntryLeader.EntityID = calculateEntityIDv1(elid, *process.EntryLeader.Start) + process.EntryLeader.EntryMeta.Type = string(entryType) +} func interactiveFromTTY(tty types.TTYDev) bool { return TTYUnknown != getTTYType(tty.Major, tty.Minor) From d7e5d994f86119f25dfa10d0ad6b5f49f93b86b6 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 10 Jun 2024 12:21:05 -0700 Subject: [PATCH 11/44] formatting --- .../processors/sessionmd/provider/provider.go | 1 - .../provider/quarkprovider/quarkprovider.go | 162 ++++++++++-------- 2 files changed, 93 insertions(+), 70 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go index abdd0d76edd..4ac9530cfea 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/provider.go @@ -9,7 +9,6 @@ package provider import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" - ) // SyncDB should ensure the DB is in a state to handle the event before returning. diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 401b42a9d58..33806d183b6 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -8,20 +8,20 @@ package quarkprovider import ( "context" + "encoding/base64" "fmt" + "os" + "os/user" + "path" + "strconv" + "strings" + "sync" "time" - "os" - "sync" - "strings" - "encoding/base64" - "strconv" - "path" - "os/user" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" "github.com/elastic/elastic-agent-libs/logp" quark "github.com/elastic/quark/go" ) @@ -150,46 +150,47 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e } func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - time.Sleep(500 * time.Millisecond) + time.Sleep(100 * time.Millisecond) return nil -//// TODO: Not working correctly... -// timeout := 5 * time.Second -// ch := time.After(timeout) -// for { -// select { -// case <- ch: -// p.logger.Errorf("%v not seen after %v", pid, timeout) -// break -// default: -// proc := p.qq.Lookup(int(pid)) -// //TODO: check event, eg. make sure exit seen when enriching exit event -// if proc != nil { -// break -// } -// time.Sleep(200 * time.Millisecond) -// } -// } + // // TODO: Not working correctly... + // + // timeout := 5 * time.Second + // ch := time.After(timeout) + // for { + // select { + // case <- ch: + // p.logger.Errorf("%v not seen after %v", pid, timeout) + // break + // default: + // proc := p.qq.Lookup(int(pid)) + // //TODO: check event, eg. make sure exit seen when enriching exit event + // if proc != nil { + // break + // } + // time.Sleep(200 * time.Millisecond) + // } + // } } func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { - qev := p.qq.Lookup(int(pid)) - if qev == nil { - return nil, fmt.Errorf("PID %d not found in cache", pid) - } + qev := p.qq.Lookup(int(pid)) + if qev == nil { + return nil, fmt.Errorf("PID %d not found in cache", pid) + } reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) ret := types.Process{ - PID: qev.Pid, - Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), - Name: basename(qev.Filename), - Executable: qev.Filename, -// Args: qev.Argv, + PID: qev.Pid, + Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Name: basename(qev.Filename), + Executable: qev.Filename, + // Args: qev.Argv, WorkingDirectory: qev.Cwd, Interactive: &interactive, } @@ -208,34 +209,35 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { } ret.TTY.CharDevice.Major = uint16(qev.Proc.TtyMajor) ret.TTY.CharDevice.Minor = uint16(qev.Proc.TtyMinor) - if qev.ExitEvent != nil { - ret.ExitCode = qev.ExitEvent.ExitCode - } + if qev.ExitEvent != nil { + ret.ExitCode = qev.ExitEvent.ExitCode + } - p.fillParent(&ret, qev.Proc.Ppid) - p.fillGroupLeader(&ret, qev.Proc.Pgid) - p.fillSessionLeader(&ret, qev.Proc.Sid) + p.fillParent(&ret, qev.Proc.Ppid) + p.fillGroupLeader(&ret, qev.Proc.Pgid) + p.fillSessionLeader(&ret, qev.Proc.Sid) + setEntityID(&ret) return &ret, nil } func (p prvdr) fillParent(process *types.Process, ppid uint32) { - qev := p.qq.Lookup(int(ppid)) - if qev == nil { - return - } + qev := p.qq.Lookup(int(ppid)) + if qev == nil { + return + } reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) euid := qev.Proc.Euid egid := qev.Proc.Egid process.Parent.PID = qev.Proc.Ppid process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.Parent.Name = basename(qev.Filename) process.Parent.Executable = qev.Filename -// process.Parent.Args = qev.Argv + // process.Parent.Args = qev.Argv process.Parent.WorkingDirectory = qev.Cwd process.Parent.Interactive = &interactive process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -250,25 +252,25 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { } } -func (p prvdr)fillGroupLeader(process *types.Process, pgid uint32) { +func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { qev := p.qq.Lookup(int(pgid)) - if qev == nil { - return - } + if qev == nil { + return + } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) euid := qev.Proc.Euid egid := qev.Proc.Egid process.GroupLeader.PID = qev.Pid process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.GroupLeader.Name = basename(qev.Filename) process.GroupLeader.Executable = qev.Filename -// process.GroupLeader.Args = qev.Argv + // process.GroupLeader.Args = qev.Argv process.GroupLeader.WorkingDirectory = qev.Cwd process.GroupLeader.Interactive = &interactive process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -285,23 +287,23 @@ func (p prvdr)fillGroupLeader(process *types.Process, pgid uint32) { func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { qev := p.qq.Lookup(int(sid)) - if qev == nil { - return - } + if qev == nil { + return + } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, - }) + Major: qev.Proc.TtyMajor, + Minor: qev.Proc.TtyMinor, + }) euid := qev.Proc.Euid egid := qev.Proc.Egid process.SessionLeader.PID = qev.Pid process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.SessionLeader.Name = basename(qev.Filename) process.SessionLeader.Executable = qev.Filename -// process.SessionLeader.Args = qev.Argv + // process.SessionLeader.Args = qev.Argv process.SessionLeader.WorkingDirectory = qev.Cwd process.SessionLeader.Interactive = &interactive process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -343,6 +345,28 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { // process.EntryLeader.EntryMeta.Type = string(entryType) //} +func setEntityID(process *types.Process) { + if process.PID != 0 && process.Start != nil { + process.EntityID = calculateEntityIDv1(process.PID, *process.Start) + } + + if process.Parent.PID != 0 && process.Parent.Start != nil { + process.Parent.EntityID = calculateEntityIDv1(process.Parent.PID, *process.Parent.Start) + } + + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + process.GroupLeader.EntityID = calculateEntityIDv1(process.GroupLeader.PID, *process.GroupLeader.Start) + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + process.SessionLeader.EntityID = calculateEntityIDv1(process.SessionLeader.PID, *process.SessionLeader.Start) + } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + process.EntryLeader.EntityID = calculateEntityIDv1(process.EntryLeader.PID, *process.EntryLeader.Start) + } +} + func interactiveFromTTY(tty types.TTYDev) bool { return TTYUnknown != getTTYType(tty.Major, tty.Minor) } From a08abf092c08382b88acd1608f4679b4c57d3091 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 11 Jun 2024 00:13:22 +0000 Subject: [PATCH 12/44] redo sync --- .../provider/quarkprovider/quarkprovider.go | 122 ++++++++++++++---- 1 file changed, 94 insertions(+), 28 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index d25dc0202cc..83eb1226281 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -134,7 +134,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e continue } for _, qev := range qevs { - logger.Debugf("qev: %v", qev) + logger.Infof("qev: %v", qev) } if len(qevs) == 0 { err = qq.Block() @@ -146,31 +146,79 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e } }(qq, logger) + bootID, _ = readBootID() + pidNsInode, _ = readPIDNsInode() + return &p, nil } +const ( + maxWaitLimit = 1200 * time.Millisecond // Maximum time SyncDB will wait for process + combinedWaitLimit = 15 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration + backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time + resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset +) + +var ( + combinedWait = 0 * time.Millisecond + inBackoff = false + backoffStart = time.Now() + since = time.Now() + backoffSkipped = 0 +) + func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - time.Sleep(100 * time.Millisecond) - return nil - - // // TODO: Not working correctly... - // - // timeout := 5 * time.Second - // ch := time.After(timeout) - // for { - // select { - // case <- ch: - // p.logger.Errorf("%v not seen after %v", pid, timeout) - // break - // default: - // proc := p.qq.Lookup(int(pid)) - // //TODO: check event, eg. make sure exit seen when enriching exit event - // if proc != nil { - // break - // } - // time.Sleep(200 * time.Millisecond) - // } - // } + if qev := p.qq.Lookup(int(pid)); qev != nil { + return nil + } + + now := time.Now() + if inBackoff { + if now.Sub(backoffStart) > backoffDuration { + p.logger.Warnf("ended backoff, skipped %d processes", backoffSkipped) + inBackoff = false + combinedWait = 0 * time.Millisecond + } else { + backoffSkipped += 1 + return nil + } + } else { + if combinedWait > combinedWaitLimit { + p.logger.Warn("starting backoff") + inBackoff = true + backoffStart = now + backoffSkipped = 0 + return nil + } + // maintain a moving window of time for the delays we track + if now.Sub(since) > resetDuration { + since = now + combinedWait = 0 * time.Millisecond + } + } + + start := now + nextWait := 5 * time.Millisecond + for { + waited := time.Since(start) + if qev := p.qq.Lookup(int(pid)); qev != nil { + p.logger.Debugf("got process that was missing after %v", waited) + combinedWait = combinedWait + waited + return nil + } + if waited >= maxWaitLimit { + e := fmt.Errorf("process %v was not seen after %v", pid, waited) + p.logger.Warnf("%w", e) + combinedWait = combinedWait + waited + return e + } + time.Sleep(nextWait) + if nextWait*2+waited > maxWaitLimit { + nextWait = maxWaitLimit - waited + } else { + nextWait = nextWait * 2 + } + } } func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { @@ -190,7 +238,7 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), Name: basename(qev.Filename), Executable: qev.Filename, - // Args: qev.Argv, + Args: []string{qev.Filename}, // TODO: Fix WorkingDirectory: qev.Cwd, Interactive: &interactive, } @@ -215,10 +263,11 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { ret.EntityID = calculateEntityIDv1(pid, *ret.Start) p.fillParent(&ret, qev.Proc.Ppid) - p.fillGroupLeader(&ret, qev.Proc.Pgid) + p.fillGroupLeader(&ret, qev.Pid) // qev.Proc.Pgid) p.fillSessionLeader(&ret, qev.Proc.Sid) p.fillEntryLeader(&ret, Init, uint32(1)) setEntityID(&ret) + setSameAsProcess(&ret) return &ret, nil } @@ -239,7 +288,7 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.Parent.Name = basename(qev.Filename) process.Parent.Executable = qev.Filename - // process.Parent.Args = qev.Argv + process.Parent.Args = []string{qev.Filename} //TODO: FIx process.Parent.WorkingDirectory = qev.Cwd process.Parent.Interactive = &interactive process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -273,7 +322,7 @@ func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.GroupLeader.Name = basename(qev.Filename) process.GroupLeader.Executable = qev.Filename - // process.GroupLeader.Args = qev.Argv + process.GroupLeader.Args = []string{qev.Filename} //TODO: fix process.GroupLeader.WorkingDirectory = qev.Cwd process.GroupLeader.Interactive = &interactive process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -307,7 +356,7 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.SessionLeader.Name = basename(qev.Filename) process.SessionLeader.Executable = qev.Filename - // process.SessionLeader.Args = qev.Argv + process.SessionLeader.Args = []string{qev.Filename} //TODO: fix process.SessionLeader.WorkingDirectory = qev.Cwd process.SessionLeader.Interactive = &interactive process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -342,7 +391,7 @@ func (p prvdr) fillEntryLeader(process *types.Process, entryType EntryType, elid process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.EntryLeader.Name = basename(qev.Filename) process.EntryLeader.Executable = qev.Filename - // process.EntryLeader.Args = qev.Argv + process.EntryLeader.Args = []string{qev.Filename} // TODO: Fix process.EntryLeader.WorkingDirectory = qev.Cwd process.EntryLeader.Interactive = &interactive process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -382,6 +431,23 @@ func setEntityID(process *types.Process) { } } +func setSameAsProcess(process *types.Process) { + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + sameAsProcess := process.PID == process.GroupLeader.PID + process.GroupLeader.SameAsProcess = &sameAsProcess + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + sameAsProcess := process.PID == process.SessionLeader.PID + process.SessionLeader.SameAsProcess = &sameAsProcess + } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + sameAsProcess := process.PID == process.EntryLeader.PID + process.EntryLeader.SameAsProcess = &sameAsProcess + } +} + func interactiveFromTTY(tty types.TTYDev) bool { return TTYUnknown != getTTYType(tty.Major, tty.Minor) } From 6019016d1eaab2f56a3bbb65d5cafbedbbd1aaa8 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 28 Jun 2024 11:42:30 -0700 Subject: [PATCH 13/44] Use quark entryleader --- .../sessionmd/provider/quarkprovider/quarkprovider.go | 8 ++++---- x-pack/auditbeat/processors/sessionmd/quark | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index d25dc0202cc..4e2ddd7b578 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -114,7 +114,7 @@ func readPIDNsInode() (uint64, error) { func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, error) { attr := quark.DefaultQueueAttr() - attr.Flags = quark.QQ_KPROBE | quark.QQ_MIN_AGG + attr.Flags = quark.QQ_KPROBE | quark.QQ_MIN_AGG | quark.QQ_ENTRY_LEADER qq, err := quark.OpenQueue(attr, 64) if err != nil { return nil, fmt.Errorf("open queue: %v", err) @@ -217,7 +217,7 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { p.fillParent(&ret, qev.Proc.Ppid) p.fillGroupLeader(&ret, qev.Proc.Pgid) p.fillSessionLeader(&ret, qev.Proc.Sid) - p.fillEntryLeader(&ret, Init, uint32(1)) + p.fillEntryLeader(&ret, qev.Proc.EntryLeaderType, qev.Proc.EntryLeader) setEntityID(&ret) return &ret, nil } @@ -323,7 +323,7 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { process.SessionLeader.EntityID = calculateEntityIDv1(sid, *process.SessionLeader.Start) } -func (p prvdr) fillEntryLeader(process *types.Process, entryType EntryType, elid uint32) { +func (p prvdr) fillEntryLeader(process *types.Process, entryType uint32, elid uint32) { qev := p.qq.Lookup(int(elid)) if qev == nil { return @@ -357,7 +357,7 @@ func (p prvdr) fillEntryLeader(process *types.Process, entryType EntryType, elid } process.EntryLeader.EntityID = calculateEntityIDv1(elid, *process.EntryLeader.Start) - process.EntryLeader.EntryMeta.Type = string(entryType) + process.EntryLeader.EntryMeta.Type = "UNKNOWN" //TODO: use real value } func setEntityID(process *types.Process) { diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark index d608a9efdf2..0ff5476ffc8 160000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -1 +1 @@ -Subproject commit d608a9efdf2bbd4dfaa693ba9f462ccb9491b691 +Subproject commit 0ff5476ffc8fa9bda7a7349de39bea7a72ba229a From 2feebe1f782dae37489f880f62cd9079c35361e3 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 28 Jun 2024 11:44:12 -0700 Subject: [PATCH 14/44] Use quark entryleader --- .../provider/quarkprovider/quarkprovider.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 19061672f9f..5ad76dd4896 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -155,8 +155,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e const ( maxWaitLimit = 1200 * time.Millisecond // Maximum time SyncDB will wait for process combinedWaitLimit = 15 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration - backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time - resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset + backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time + resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset ) var ( @@ -234,11 +234,11 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { }) ret := types.Process{ - PID: qev.Pid, - Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), - Name: basename(qev.Filename), - Executable: qev.Filename, - Args: []string{qev.Filename}, // TODO: Fix + PID: qev.Pid, + Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Name: basename(qev.Filename), + Executable: qev.Filename, + Args: []string{qev.Filename}, // TODO: Fix WorkingDirectory: qev.Cwd, Interactive: &interactive, } From 298d59cdfaec5de83916d59083137dfdec5cce82 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 28 Jun 2024 22:26:18 +0000 Subject: [PATCH 15/44] Send end time --- .../provider/quarkprovider/quarkprovider.go | 24 ++++++++++++++++++- .../processors/sessionmd/types/process.go | 4 ++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 5ad76dd4896..1cbf97e43a4 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -259,6 +259,7 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { ret.TTY.CharDevice.Minor = uint16(qev.Proc.TtyMinor) if qev.ExitEvent != nil { ret.ExitCode = qev.ExitEvent.ExitCode + ret.End = timeutils.TimeFromNsSinceBoot(time.Duration(qev.ExitEvent.ExitTimeEvent)) } ret.EntityID = calculateEntityIDv1(pid, *ret.Start) @@ -406,7 +407,7 @@ func (p prvdr) fillEntryLeader(process *types.Process, entryType uint32, elid ui } process.EntryLeader.EntityID = calculateEntityIDv1(elid, *process.EntryLeader.Start) - process.EntryLeader.EntryMeta.Type = "UNKNOWN" //TODO: use real value + process.EntryLeader.EntryMeta.Type = getEntryTypeName(qev.Proc.EntryLeaderType) } func setEntityID(process *types.Process) { @@ -508,3 +509,24 @@ func getGroupName(id string) (string, bool) { } return group.Name, true } + +func getEntryTypeName(entryType uint32) string { + switch int(entryType) { + case quark.QUARK_ELT_INIT: + return "init" + case quark.QUARK_ELT_SSHD: + return "sshd" + case quark.QUARK_ELT_SSM: + return "ssm" + case quark.QUARK_ELT_CONTAINER: + return "container" + case quark.QUARK_ELT_TERM: + return "terminal" + case quark.QUARK_ELT_CONSOLE: + return "console" + case quark.QUARK_ELT_KTHREAD: + return "kthread" + default: + return "unknown" + } +} diff --git a/x-pack/auditbeat/processors/sessionmd/types/process.go b/x-pack/auditbeat/processors/sessionmd/types/process.go index 8f52a9c5aa5..ee05206636d 100644 --- a/x-pack/auditbeat/processors/sessionmd/types/process.go +++ b/x-pack/auditbeat/processors/sessionmd/types/process.go @@ -448,6 +448,10 @@ func (p *Process) ToMap() mapstr.M { if p.EntryLeader.Start != nil { process.Put("entry_leader.start", p.EntryLeader.Start) } + if p.End != nil { + process.Put("end", p.End) + } + // TODO: are other Ends needed, ancestors shouldn't end before process return process } From 4dfeb9fa3fce080e3a3b4ae9e78ead25330736ce Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 2 Jul 2024 21:38:38 +0000 Subject: [PATCH 16/44] Update timestamp calculations --- .../provider/quarkprovider/quarkprovider.go | 33 +++++++++---------- x-pack/auditbeat/processors/sessionmd/quark | 2 +- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 1cbf97e43a4..206c6b6e630 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -20,7 +20,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" quark "github.com/elastic/quark/go" @@ -227,15 +226,16 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { return nil, fmt.Errorf("PID %d not found in cache", pid) } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) interactive := interactiveFromTTY(types.TTYDev{ Major: qev.Proc.TtyMajor, Minor: qev.Proc.TtyMinor, }) + start := time.Unix(0, int64(qev.Proc.TimeBoot)) + ret := types.Process{ PID: qev.Pid, - Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Start: &start, Name: basename(qev.Filename), Executable: qev.Filename, Args: []string{qev.Filename}, // TODO: Fix @@ -258,15 +258,16 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { ret.TTY.CharDevice.Major = uint16(qev.Proc.TtyMajor) ret.TTY.CharDevice.Minor = uint16(qev.Proc.TtyMinor) if qev.ExitEvent != nil { + end := time.Unix(0, int64(qev.ExitEvent.ExitTimeEvent)) ret.ExitCode = qev.ExitEvent.ExitCode - ret.End = timeutils.TimeFromNsSinceBoot(time.Duration(qev.ExitEvent.ExitTimeEvent)) + ret.End = &end } ret.EntityID = calculateEntityIDv1(pid, *ret.Start) p.fillParent(&ret, qev.Proc.Ppid) p.fillGroupLeader(&ret, qev.Pid) // qev.Proc.Pgid) p.fillSessionLeader(&ret, qev.Proc.Sid) - p.fillEntryLeader(&ret, qev.Proc.EntryLeaderType, qev.Proc.EntryLeader) + p.fillEntryLeader(&ret, qev.Proc.EntryLeader) setEntityID(&ret) setSameAsProcess(&ret) return &ret, nil @@ -278,7 +279,7 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { return } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + start := time.Unix(0, int64(qev.Proc.TimeBoot)) interactive := interactiveFromTTY(types.TTYDev{ Major: qev.Proc.TtyMajor, Minor: qev.Proc.TtyMinor, @@ -286,7 +287,7 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { euid := qev.Proc.Euid egid := qev.Proc.Egid process.Parent.PID = qev.Proc.Ppid - process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.Parent.Start = &start process.Parent.Name = basename(qev.Filename) process.Parent.Executable = qev.Filename process.Parent.Args = []string{qev.Filename} //TODO: FIx @@ -311,7 +312,7 @@ func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { return } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + start := time.Unix(0, int64(qev.Proc.TimeBoot)) interactive := interactiveFromTTY(types.TTYDev{ Major: qev.Proc.TtyMajor, @@ -320,7 +321,7 @@ func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { euid := qev.Proc.Euid egid := qev.Proc.Egid process.GroupLeader.PID = qev.Pid - process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.GroupLeader.Start = &start process.GroupLeader.Name = basename(qev.Filename) process.GroupLeader.Executable = qev.Filename process.GroupLeader.Args = []string{qev.Filename} //TODO: fix @@ -345,7 +346,7 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { return } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + start := time.Unix(0, int64(qev.Proc.TimeBoot)) interactive := interactiveFromTTY(types.TTYDev{ Major: qev.Proc.TtyMajor, @@ -354,7 +355,7 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { euid := qev.Proc.Euid egid := qev.Proc.Egid process.SessionLeader.PID = qev.Pid - process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.SessionLeader.Start = &start process.SessionLeader.Name = basename(qev.Filename) process.SessionLeader.Executable = qev.Filename process.SessionLeader.Args = []string{qev.Filename} //TODO: fix @@ -373,13 +374,14 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { process.SessionLeader.EntityID = calculateEntityIDv1(sid, *process.SessionLeader.Start) } -func (p prvdr) fillEntryLeader(process *types.Process, entryType uint32, elid uint32) { +func (p prvdr) fillEntryLeader(process *types.Process, elid uint32) { qev := p.qq.Lookup(int(elid)) if qev == nil { return } - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(qev.Proc.TimeBoot) + start := time.Unix(0, int64(qev.Proc.TimeBoot)) + interactive := interactiveFromTTY(types.TTYDev{ Major: qev.Proc.TtyMajor, @@ -389,10 +391,7 @@ func (p prvdr) fillEntryLeader(process *types.Process, entryType uint32, elid ui euid := qev.Proc.Euid egid := qev.Proc.Egid process.EntryLeader.PID = qev.Pid - process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.EntryLeader.Name = basename(qev.Filename) - process.EntryLeader.Executable = qev.Filename - process.EntryLeader.Args = []string{qev.Filename} // TODO: Fix + process.EntryLeader.Start = &start process.EntryLeader.WorkingDirectory = qev.Cwd process.EntryLeader.Interactive = &interactive process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark index 0ff5476ffc8..b171c6cdec8 160000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -1 +1 @@ -Subproject commit 0ff5476ffc8fa9bda7a7349de39bea7a72ba229a +Subproject commit b171c6cdec8cc53c91a63df2929c9ecfeb8fbf5e From 3fa6dd3b962ee60bbf78a270d712a4d5bdeb2c10 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 13 Sep 2024 09:59:17 -0700 Subject: [PATCH 17/44] update quark ref --- x-pack/auditbeat/processors/sessionmd/quark | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark index b171c6cdec8..72525ccc03c 160000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ b/x-pack/auditbeat/processors/sessionmd/quark @@ -1 +1 @@ -Subproject commit b171c6cdec8cc53c91a63df2929c9ecfeb8fbf5e +Subproject commit 72525ccc03c9b0ceeeae6aa6c65994a2f12c5517 From 590c0eb95582a2677b1678d82202c665228c2a16 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 13 Sep 2024 14:49:58 -0700 Subject: [PATCH 18/44] Use new quark version --- .../provider/quarkprovider/quarkprovider.go | 146 +++++++++--------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 206c6b6e630..8cda560db2b 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -127,15 +127,15 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e go func(qq *quark.Queue, logger *logp.Logger) { for { - qevs, err := qq.GetEvents() + procs, err := qq.GetEvents() if err != nil { logger.Errorf("get events from quark: %v", err) continue } - for _, qev := range qevs { - logger.Infof("qev: %v", qev) + for _, proc := range procs { + logger.Infof("proc: %v", proc) } - if len(qevs) == 0 { + if len(procs) == 0 { err = qq.Block() if err != nil { logger.Errorf("quark block: %v", err) @@ -167,7 +167,7 @@ var ( ) func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - if qev := p.qq.Lookup(int(pid)); qev != nil { + if _, found := p.qq.Lookup(int(pid)); found { return nil } @@ -200,7 +200,7 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { nextWait := 5 * time.Millisecond for { waited := time.Since(start) - if qev := p.qq.Lookup(int(pid)); qev != nil { + if _, found := p.qq.Lookup(int(pid)); found { p.logger.Debugf("got process that was missing after %v", waited) combinedWait = combinedWait + waited return nil @@ -221,30 +221,30 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { } func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { - qev := p.qq.Lookup(int(pid)) - if qev == nil { + proc, found := p.qq.Lookup(int(pid)) + if !found { return nil, fmt.Errorf("PID %d not found in cache", pid) } interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, + Major: proc.Proc.TtyMajor, + Minor: proc.Proc.TtyMinor, }) - start := time.Unix(0, int64(qev.Proc.TimeBoot)) + start := time.Unix(0, int64(proc.Proc.TimeBoot)) ret := types.Process{ - PID: qev.Pid, + PID: proc.Pid, Start: &start, - Name: basename(qev.Filename), - Executable: qev.Filename, - Args: []string{qev.Filename}, // TODO: Fix - WorkingDirectory: qev.Cwd, + Name: basename(proc.Filename), + Executable: proc.Filename, + Args: []string{proc.Filename}, // TODO: Fix + WorkingDirectory: proc.Cwd, Interactive: &interactive, } - euid := qev.Proc.Euid - egid := qev.Proc.Egid + euid := proc.Proc.Euid + egid := proc.Proc.Egid ret.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(ret.User.ID) if ok { @@ -255,43 +255,43 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { if ok { ret.Group.Name = groupname } - ret.TTY.CharDevice.Major = uint16(qev.Proc.TtyMajor) - ret.TTY.CharDevice.Minor = uint16(qev.Proc.TtyMinor) - if qev.ExitEvent != nil { - end := time.Unix(0, int64(qev.ExitEvent.ExitTimeEvent)) - ret.ExitCode = qev.ExitEvent.ExitCode + ret.TTY.CharDevice.Major = uint16(proc.Proc.TtyMajor) + ret.TTY.CharDevice.Minor = uint16(proc.Proc.TtyMinor) + if proc.Exit.Valid { + end := time.Unix(0, int64(proc.Exit.ExitTimeProcess)) + ret.ExitCode = proc.Exit.ExitCode ret.End = &end } ret.EntityID = calculateEntityIDv1(pid, *ret.Start) - p.fillParent(&ret, qev.Proc.Ppid) - p.fillGroupLeader(&ret, qev.Pid) // qev.Proc.Pgid) - p.fillSessionLeader(&ret, qev.Proc.Sid) - p.fillEntryLeader(&ret, qev.Proc.EntryLeader) + p.fillParent(&ret, proc.Proc.Ppid) + p.fillGroupLeader(&ret, proc.Pid) // proc.Proc.Pgid) + p.fillSessionLeader(&ret, proc.Proc.Sid) + p.fillEntryLeader(&ret, proc.Proc.EntryLeader) setEntityID(&ret) setSameAsProcess(&ret) return &ret, nil } func (p prvdr) fillParent(process *types.Process, ppid uint32) { - qev := p.qq.Lookup(int(ppid)) - if qev == nil { + proc, found := p.qq.Lookup(int(ppid)) + if !found { return } - start := time.Unix(0, int64(qev.Proc.TimeBoot)) + start := time.Unix(0, int64(proc.Proc.TimeBoot)) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, + Major: proc.Proc.TtyMajor, + Minor: proc.Proc.TtyMinor, }) - euid := qev.Proc.Euid - egid := qev.Proc.Egid - process.Parent.PID = qev.Proc.Ppid + euid := proc.Proc.Euid + egid := proc.Proc.Egid + process.Parent.PID = proc.Proc.Ppid process.Parent.Start = &start - process.Parent.Name = basename(qev.Filename) - process.Parent.Executable = qev.Filename - process.Parent.Args = []string{qev.Filename} //TODO: FIx - process.Parent.WorkingDirectory = qev.Cwd + process.Parent.Name = basename(proc.Filename) + process.Parent.Executable = proc.Filename + process.Parent.Args = []string{proc.Filename} //TODO: FIx + process.Parent.WorkingDirectory = proc.Cwd process.Parent.Interactive = &interactive process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(process.Parent.User.ID) @@ -307,25 +307,25 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { } func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { - qev := p.qq.Lookup(int(pgid)) - if qev == nil { + proc, found := p.qq.Lookup(int(pgid)) + if !found { return } - start := time.Unix(0, int64(qev.Proc.TimeBoot)) + start := time.Unix(0, int64(proc.Proc.TimeBoot)) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, + Major: proc.Proc.TtyMajor, + Minor: proc.Proc.TtyMinor, }) - euid := qev.Proc.Euid - egid := qev.Proc.Egid - process.GroupLeader.PID = qev.Pid + euid := proc.Proc.Euid + egid := proc.Proc.Egid + process.GroupLeader.PID = proc.Pid process.GroupLeader.Start = &start - process.GroupLeader.Name = basename(qev.Filename) - process.GroupLeader.Executable = qev.Filename - process.GroupLeader.Args = []string{qev.Filename} //TODO: fix - process.GroupLeader.WorkingDirectory = qev.Cwd + process.GroupLeader.Name = basename(proc.Filename) + process.GroupLeader.Executable = proc.Filename + process.GroupLeader.Args = []string{proc.Filename} //TODO: fix + process.GroupLeader.WorkingDirectory = proc.Cwd process.GroupLeader.Interactive = &interactive process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(process.GroupLeader.User.ID) @@ -341,25 +341,25 @@ func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { } func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { - qev := p.qq.Lookup(int(sid)) - if qev == nil { + proc, found := p.qq.Lookup(int(sid)) + if !found { return } - start := time.Unix(0, int64(qev.Proc.TimeBoot)) + start := time.Unix(0, int64(proc.Proc.TimeBoot)) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, + Major: proc.Proc.TtyMajor, + Minor: proc.Proc.TtyMinor, }) - euid := qev.Proc.Euid - egid := qev.Proc.Egid - process.SessionLeader.PID = qev.Pid + euid := proc.Proc.Euid + egid := proc.Proc.Egid + process.SessionLeader.PID = proc.Pid process.SessionLeader.Start = &start - process.SessionLeader.Name = basename(qev.Filename) - process.SessionLeader.Executable = qev.Filename - process.SessionLeader.Args = []string{qev.Filename} //TODO: fix - process.SessionLeader.WorkingDirectory = qev.Cwd + process.SessionLeader.Name = basename(proc.Filename) + process.SessionLeader.Executable = proc.Filename + process.SessionLeader.Args = []string{proc.Filename} //TODO: fix + process.SessionLeader.WorkingDirectory = proc.Cwd process.SessionLeader.Interactive = &interactive process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(process.SessionLeader.User.ID) @@ -375,24 +375,24 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { } func (p prvdr) fillEntryLeader(process *types.Process, elid uint32) { - qev := p.qq.Lookup(int(elid)) - if qev == nil { + proc, found := p.qq.Lookup(int(elid)) + if !found { return } - start := time.Unix(0, int64(qev.Proc.TimeBoot)) + start := time.Unix(0, int64(proc.Proc.TimeBoot)) interactive := interactiveFromTTY(types.TTYDev{ - Major: qev.Proc.TtyMajor, - Minor: qev.Proc.TtyMinor, + Major: proc.Proc.TtyMajor, + Minor: proc.Proc.TtyMinor, }) - euid := qev.Proc.Euid - egid := qev.Proc.Egid - process.EntryLeader.PID = qev.Pid + euid := proc.Proc.Euid + egid := proc.Proc.Egid + process.EntryLeader.PID = proc.Pid process.EntryLeader.Start = &start - process.EntryLeader.WorkingDirectory = qev.Cwd + process.EntryLeader.WorkingDirectory = proc.Cwd process.EntryLeader.Interactive = &interactive process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) username, ok := getUserName(process.EntryLeader.User.ID) @@ -406,7 +406,7 @@ func (p prvdr) fillEntryLeader(process *types.Process, elid uint32) { } process.EntryLeader.EntityID = calculateEntityIDv1(elid, *process.EntryLeader.Start) - process.EntryLeader.EntryMeta.Type = getEntryTypeName(qev.Proc.EntryLeaderType) + process.EntryLeader.EntryMeta.Type = getEntryTypeName(proc.Proc.EntryLeaderType) } func setEntityID(process *types.Process) { From 5cca22f20a3c9c1bc91bb6aef0d62b74ead8ef3f Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 17 Sep 2024 16:41:08 -0700 Subject: [PATCH 19/44] protect quark calls with lock --- .../sessionmd/add_session_metadata.go | 1 - .../provider/quarkprovider/quarkprovider.go | 35 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 7c73a2520d9..cab991626b5 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -94,7 +94,6 @@ func New(cfg *cfg.C) (beat.Processor, error) { if err != nil { return nil, fmt.Errorf("failed to create quark provider: %w", err) } - db.Close() // db not used with quark default: return nil, fmt.Errorf("unknown backend configuration") } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 8cda560db2b..3615c760c2c 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -29,6 +29,7 @@ type prvdr struct { ctx context.Context logger *logp.Logger qq *quark.Queue + qqMtx *sync.Mutex } type TTYType int @@ -119,15 +120,19 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e return nil, fmt.Errorf("open queue: %v", err) } + var qqMtx sync.Mutex p := prvdr{ ctx: ctx, logger: logger, qq: qq, + qqMtx: &qqMtx, } - go func(qq *quark.Queue, logger *logp.Logger) { + go func(qq *quark.Queue, logger *logp.Logger, p *prvdr) { for { + p.qqMtx.Lock() procs, err := qq.GetEvents() + p.qqMtx.Unlock() if err != nil { logger.Errorf("get events from quark: %v", err) continue @@ -143,7 +148,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e } } } - }(qq, logger) + }(qq, logger, &p) bootID, _ = readBootID() pidNsInode, _ = readPIDNsInode() @@ -167,7 +172,7 @@ var ( ) func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - if _, found := p.qq.Lookup(int(pid)); found { + if _, found := p.lookupLocked(pid); found { return nil } @@ -200,7 +205,7 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { nextWait := 5 * time.Millisecond for { waited := time.Since(start) - if _, found := p.qq.Lookup(int(pid)); found { + if _, found := p.lookupLocked(pid); found { p.logger.Debugf("got process that was missing after %v", waited) combinedWait = combinedWait + waited return nil @@ -221,7 +226,7 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { } func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { - proc, found := p.qq.Lookup(int(pid)) + proc, found := p.lookupLocked(pid) if !found { return nil, fmt.Errorf("PID %d not found in cache", pid) } @@ -238,7 +243,7 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { Start: &start, Name: basename(proc.Filename), Executable: proc.Filename, - Args: []string{proc.Filename}, // TODO: Fix + Args: proc.Cmdline, WorkingDirectory: proc.Cwd, Interactive: &interactive, } @@ -265,7 +270,7 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { ret.EntityID = calculateEntityIDv1(pid, *ret.Start) p.fillParent(&ret, proc.Proc.Ppid) - p.fillGroupLeader(&ret, proc.Pid) // proc.Proc.Pgid) + p.fillGroupLeader(&ret, proc.Proc.Pgid) p.fillSessionLeader(&ret, proc.Proc.Sid) p.fillEntryLeader(&ret, proc.Proc.EntryLeader) setEntityID(&ret) @@ -273,8 +278,15 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { return &ret, nil } +func (p prvdr) lookupLocked(pid uint32) (quark.Process, bool) { + p.qqMtx.Lock() + p.qqMtx.Unlock() + + return p.qq.Lookup(int(pid)) +} + func (p prvdr) fillParent(process *types.Process, ppid uint32) { - proc, found := p.qq.Lookup(int(ppid)) + proc, found := p.lookupLocked(ppid) if !found { return } @@ -307,7 +319,7 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { } func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { - proc, found := p.qq.Lookup(int(pgid)) + proc, found := p.lookupLocked(pgid) if !found { return } @@ -341,7 +353,7 @@ func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { } func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { - proc, found := p.qq.Lookup(int(sid)) + proc, found := p.lookupLocked(sid) if !found { return } @@ -375,14 +387,13 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { } func (p prvdr) fillEntryLeader(process *types.Process, elid uint32) { - proc, found := p.qq.Lookup(int(elid)) + proc, found := p.lookupLocked(elid) if !found { return } start := time.Unix(0, int64(proc.Proc.TimeBoot)) - interactive := interactiveFromTTY(types.TTYDev{ Major: proc.Proc.TtyMajor, Minor: proc.Proc.TtyMinor, From 743d15f350abcb80a6274bd440ee820954c2cfba Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 18 Sep 2024 20:16:30 +0000 Subject: [PATCH 20/44] Set more things correctly --- .../sessionmd/provider/quarkprovider/quarkprovider.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go index 3615c760c2c..db50b6929b0 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go @@ -298,11 +298,11 @@ func (p prvdr) fillParent(process *types.Process, ppid uint32) { }) euid := proc.Proc.Euid egid := proc.Proc.Egid - process.Parent.PID = proc.Proc.Ppid + process.Parent.PID = proc.Pid process.Parent.Start = &start process.Parent.Name = basename(proc.Filename) process.Parent.Executable = proc.Filename - process.Parent.Args = []string{proc.Filename} //TODO: FIx + process.Parent.Args = proc.Cmdline process.Parent.WorkingDirectory = proc.Cwd process.Parent.Interactive = &interactive process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -336,7 +336,7 @@ func (p prvdr) fillGroupLeader(process *types.Process, pgid uint32) { process.GroupLeader.Start = &start process.GroupLeader.Name = basename(proc.Filename) process.GroupLeader.Executable = proc.Filename - process.GroupLeader.Args = []string{proc.Filename} //TODO: fix + process.GroupLeader.Args = proc.Cmdline process.GroupLeader.WorkingDirectory = proc.Cwd process.GroupLeader.Interactive = &interactive process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) @@ -370,7 +370,7 @@ func (p prvdr) fillSessionLeader(process *types.Process, sid uint32) { process.SessionLeader.Start = &start process.SessionLeader.Name = basename(proc.Filename) process.SessionLeader.Executable = proc.Filename - process.SessionLeader.Args = []string{proc.Filename} //TODO: fix + process.SessionLeader.Args = proc.Cmdline process.SessionLeader.WorkingDirectory = proc.Cwd process.SessionLeader.Interactive = &interactive process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) From cb255ebccc9f97932266c40f6ace200b92751dca Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 18 Sep 2024 13:36:46 -0700 Subject: [PATCH 21/44] Replace legacy ebpf provider with modern provider --- .../sessionmd/add_session_metadata.go | 26 +- .../docs/add_session_metadata.asciidoc | 34 +-- .../provider/ebpfprovider/ebpfprovider.go | 235 ------------------ .../modernprovider.go} | 2 +- 4 files changed, 28 insertions(+), 269 deletions(-) delete mode 100644 x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go rename x-pack/auditbeat/processors/sessionmd/provider/{quarkprovider/quarkprovider.go => modernprovider/modernprovider.go} (99%) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index cab991626b5..bbe1b103e7f 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -17,9 +17,8 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/modernprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -59,7 +58,7 @@ func New(cfg *cfg.C) (beat.Processor, error) { return nil, fmt.Errorf("failed to create DB: %w", err) } - if c.Backend != "quark" { + if c.Backend != "modern" { backfilledPIDs := db.ScrapeProcfs() logger.Infof("backfilled %d processes", len(backfilledPIDs)) } @@ -68,31 +67,26 @@ func New(cfg *cfg.C) (beat.Processor, error) { switch c.Backend { case "auto": - p, err = ebpfprovider.NewProvider(ctx, logger, db) + p, err = modernprovider.NewProvider(ctx, logger) if err != nil { - // Most likely cause of error is not supporting ebpf on system, try procfs + // Most likely cause of error is not supporting ebpf or kprobes on system, try procfs p, err = procfsprovider.NewProvider(ctx, logger, db, reader, c.PIDField) if err != nil { return nil, fmt.Errorf("failed to create provider: %w", err) } logger.Info("backend=auto using procfs") } else { - logger.Info("backend=auto using ebpf") - } - case "ebpf": - p, err = ebpfprovider.NewProvider(ctx, logger, db) - if err != nil { - return nil, fmt.Errorf("failed to create ebpf provider: %w", err) + logger.Info("backend=auto using modern") } case "procfs": p, err = procfsprovider.NewProvider(ctx, logger, db, reader, c.PIDField) if err != nil { return nil, fmt.Errorf("failed to create procfs provider: %w", err) } - case "quark": - p, err = quarkprovider.NewProvider(ctx, logger) + case "modern": + p, err = modernprovider.NewProvider(ctx, logger) if err != nil { - return nil, fmt.Errorf("failed to create quark provider: %w", err) + return nil, fmt.Errorf("failed to create modern provider: %w", err) } default: return nil, fmt.Errorf("unknown backend configuration") @@ -157,8 +151,8 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { } var fullProcess types.Process - if p.backend == "quark" { - // Quark doesn't enrich with the processor DB; process info is taken directly from quark cache + if p.backend == "modern" { + // modern doesn't enrich with the processor DB; process info is taken directly from modern cache proc, err := p.provider.GetProcess(pid) if err != nil { e := fmt.Errorf("pid %v not found in db: %w", pid, err) diff --git a/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc b/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc index d29c5d0ac80..8f79227c229 100644 --- a/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc +++ b/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc @@ -8,7 +8,7 @@ beta::[] The `add_session_metadata` processor enriches process events with additional information that users can see using the {security-guide}/session-view.html[Session View] tool in the -{elastic-sec} platform. +{elastic-sec} platform. NOTE: The current release of `add_session_metadata` processor for {auditbeat} is limited to virtual machines (VMs) and bare metal environments. @@ -27,9 +27,9 @@ auditbeat.modules: [[add-session-metadata-explained]] ==== How the `add_session_metadata` processor works -Using the available Linux kernel technology, the processor collects comprehensive information on all running system processes, compiling this data into a process database. -When processing an event (such as those generated by the {auditbeat} `auditd` module), the processor queries this database to retrieve information about related processes, including the parent process, session leader, process group leader, and entry leader. -It then enriches the original event with this metadata, providing a more complete picture of process relationships and system activities. +Using the available Linux kernel technology, the processor collects comprehensive information on all running system processes, compiling this data into a process database. +When processing an event (such as those generated by the {auditbeat} `auditd` module), the processor queries this database to retrieve information about related processes, including the parent process, session leader, process group leader, and entry leader. +It then enriches the original event with this metadata, providing a more complete picture of process relationships and system activities. This enhanced data enables the powerful {security-guide}/session-view.html[Session View] tool in the {elastic-sec} platform, offering users deeper insights for analysis and investigation. @@ -40,17 +40,17 @@ This enhanced data enables the powerful {security-guide}/session-view.html[Sessi The `add_session_metadata` processor operates using various backend options. * `auto` is the recommended setting. - It attempts to use `ebpf` first, falling back to `procfs` if necessary, ensuring compatibility even on systems without `ebpf` support. -* `ebpf` collects process information with eBPF. - This backend requires a system with Linux kernel 5.10.16 or above, kernel support for eBPF enabled, and auditbeat running as superuser. -* `procfs` collects process information with the proc filesystem. - This is compatible with older systems that may not support ebpf. - To gather complete process info, auditbeat requires permissions to read all process data in procfs; for example, run as a superuser or have the `SYS_PTRACE` capability. + It attempts to use `modern` first, falling back to `procfs` if necessary, ensuring compatibility even on systems without `modern` support. +* `modern` collects process information with eBPF or kprobes. + This backend requires a system with Linux kernel 5.10.16 or above, kernel support for eBPF enabled, and auditbeat running as superuser. //TODO: ***Update requirements*** +* `procfs` collects process information with the proc filesystem. + This is compatible with older systems that may not support ebpf. + To gather complete process info, auditbeat requires permissions to read all process data in procfs; for example, run as a superuser or have the `SYS_PTRACE` capability. [[add-session-metadata-containers]] ===== Containers -If you are running {auditbeat} in a container, the container must run in the host's PID namespace. -With the `auto` or `ebpf` backend, these host directories must also be mounted to the same path within the container: `/sys/kernel/debug`, `/sys/fs/bpf`. +If you are running {auditbeat} in a container, the container must run in the host's PID namespace. +With the `auto` or `modern` backend, these host directories must also be mounted to the same path within the container: `/sys/kernel/debug`, `/sys/fs/bpf`. [[add-session-metadata-enable]] ==== Enable and configure Session View in {auditbeat} @@ -58,10 +58,10 @@ With the `auto` or `ebpf` backend, these host directories must also be mounted t To configure and enable {security-guide}/session-view.html[Session View] functionality, you'll: * Add the `add_sessions_metadata` processor to your `auditbeat.yml` file. -* Configure audit rules in your `auditbeat.yml` file. +* Configure audit rules in your `auditbeat.yml` file. * Restart {auditbeat}. -We'll walk you through these steps in more detail. +We'll walk you through these steps in more detail. . Edit your `auditbeat.yml` file and add this info to the modules configuration section: + @@ -89,11 +89,11 @@ auditbeat.modules: -a always,exit -F arch=b64 -S setsid ------------------------------------- + -. Save your configuration changes. +. Save your configuration changes. + -. Restart {auditbeat}: +. Restart {auditbeat}: + [source,sh] ------------------------------------- sudo systemctl restart auditbeat -------------------------------------- \ No newline at end of file +------------------------------------- diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go deleted file mode 100644 index 8b780bc16f5..00000000000 --- a/x-pack/auditbeat/processors/sessionmd/provider/ebpfprovider/ebpfprovider.go +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build linux - -package ebpfprovider - -import ( - "context" - "fmt" - "time" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/ebpf" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" - "github.com/elastic/ebpfevents" - "github.com/elastic/elastic-agent-libs/logp" -) - -const ( - name = "add_session_metadata" - eventMask = ebpf.EventMask(ebpfevents.EventTypeProcessFork | ebpfevents.EventTypeProcessExec | ebpfevents.EventTypeProcessExit) -) - -type prvdr struct { - ctx context.Context - logger *logp.Logger - db *processdb.DB -} - -func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { - p := prvdr{ - ctx: ctx, - logger: logger, - db: db, - } - - w, err := ebpf.GetWatcher() - if err != nil { - return nil, fmt.Errorf("get ebpf watcher: %w", err) - } - - records := w.Subscribe(name, eventMask) - - go func(logger logp.Logger) { - for { - r := <-records - if r.Error != nil { - logger.Warnw("received error from the ebpf subscription", "error", err) - continue - } - if r.Event == nil { - continue - } - ev := r.Event - switch ev.Type { - case ebpfevents.EventTypeProcessFork: - body, ok := ev.Body.(*ebpfevents.ProcessFork) - if !ok { - logger.Errorf("unexpected event body, got %T", ev.Body) - continue - } - pe := types.ProcessForkEvent{ - ParentPIDs: types.PIDInfo{ - Tid: body.ParentPids.Tid, - Tgid: body.ParentPids.Tgid, - Ppid: body.ParentPids.Ppid, - Pgid: body.ParentPids.Pgid, - Sid: body.ParentPids.Sid, - StartTimeNS: body.ParentPids.StartTimeNs, - }, - ChildPIDs: types.PIDInfo{ - Tid: body.ChildPids.Tid, - Tgid: body.ChildPids.Tgid, - Ppid: body.ChildPids.Ppid, - Pgid: body.ChildPids.Pgid, - Sid: body.ChildPids.Sid, - StartTimeNS: body.ChildPids.StartTimeNs, - }, - Creds: types.CredInfo{ - Ruid: body.Creds.Ruid, - Rgid: body.Creds.Rgid, - Euid: body.Creds.Euid, - Egid: body.Creds.Egid, - Suid: body.Creds.Suid, - Sgid: body.Creds.Sgid, - CapPermitted: body.Creds.CapPermitted, - CapEffective: body.Creds.CapEffective, - }, - } - p.db.InsertFork(pe) - case ebpfevents.EventTypeProcessExec: - body, ok := ev.Body.(*ebpfevents.ProcessExec) - if !ok { - logger.Errorf("unexpected event body") - continue - } - pe := types.ProcessExecEvent{ - PIDs: types.PIDInfo{ - Tid: body.Pids.Tid, - Tgid: body.Pids.Tgid, - Ppid: body.Pids.Ppid, - Pgid: body.Pids.Pgid, - Sid: body.Pids.Sid, - StartTimeNS: body.Pids.StartTimeNs, - }, - Creds: types.CredInfo{ - Ruid: body.Creds.Ruid, - Rgid: body.Creds.Rgid, - Euid: body.Creds.Euid, - Egid: body.Creds.Egid, - Suid: body.Creds.Suid, - Sgid: body.Creds.Sgid, - CapPermitted: body.Creds.CapPermitted, - CapEffective: body.Creds.CapEffective, - }, - CTTY: types.TTYDev{ - Major: uint32(body.CTTY.Major), - Minor: uint32(body.CTTY.Minor), - }, - CWD: body.Cwd, - Argv: body.Argv, - Env: body.Env, - Filename: body.Filename, - } - p.db.InsertExec(pe) - case ebpfevents.EventTypeProcessExit: - body, ok := ev.Body.(*ebpfevents.ProcessExit) - if !ok { - logger.Errorf("unexpected event body") - continue - } - pe := types.ProcessExitEvent{ - PIDs: types.PIDInfo{ - Tid: body.Pids.Tid, - Tgid: body.Pids.Tgid, - Ppid: body.Pids.Ppid, - Pgid: body.Pids.Pgid, - Sid: body.Pids.Sid, - StartTimeNS: body.Pids.StartTimeNs, - }, - ExitCode: body.ExitCode, - } - p.db.InsertExit(pe) - } - } - }(*p.logger) - - return &p, nil -} - -func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { - return nil, fmt.Errorf("not implemented") -} - -const ( - maxWaitLimit = 200 * time.Millisecond // Maximum time SyncDB will wait for process - combinedWaitLimit = 2 * time.Second // Multiple SyncDB calls will wait up to this amount within resetDuration - backoffDuration = 10 * time.Second // SyncDB will stop waiting for processes for this time - resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset -) - -var ( - combinedWait = 0 * time.Millisecond - inBackoff = false - backoffStart = time.Now() - since = time.Now() - backoffSkipped = 0 -) - -// With ebpf, process events are pushed to the DB by the above goroutine, so this doesn't actually update the DB. -// It does to try sync the processor and ebpf events, so that the process is in the process db before continuing. -// -// It's possible that the event to enrich arrives before the process is inserted into the DB. In that case, this -// will block continuing the enrichment until the process is seen (or the timeout is reached). -// -// If for some reason a lot of time has been spent waiting for missing processes, this also has a backoff timer during -// which it will continue without waiting for missing events to arrive, so the processor doesn't become overly backed-up -// waiting for these processes, at the cost of possibly not enriching some processes. -func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - if p.db.HasProcess(pid) { - return nil - } - - now := time.Now() - if inBackoff { - if now.Sub(backoffStart) > backoffDuration { - p.logger.Warnf("ended backoff, skipped %d processes", backoffSkipped) - inBackoff = false - combinedWait = 0 * time.Millisecond - } else { - backoffSkipped += 1 - return nil - } - } else { - if combinedWait > combinedWaitLimit { - p.logger.Warn("starting backoff") - inBackoff = true - backoffStart = now - backoffSkipped = 0 - return nil - } - // maintain a moving window of time for the delays we track - if now.Sub(since) > resetDuration { - since = now - combinedWait = 0 * time.Millisecond - } - } - - start := now - nextWait := 5 * time.Millisecond - for { - waited := time.Since(start) - if p.db.HasProcess(pid) { - p.logger.Debugf("got process that was missing after %v", waited) - combinedWait = combinedWait + waited - return nil - } - if waited >= maxWaitLimit { - e := fmt.Errorf("process %v was not seen after %v", pid, waited) - p.logger.Warnf("%w", e) - combinedWait = combinedWait + waited - return e - } - time.Sleep(nextWait) - if nextWait*2+waited > maxWaitLimit { - nextWait = maxWaitLimit - waited - } else { - nextWait = nextWait * 2 - } - } -} diff --git a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go similarity index 99% rename from x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go rename to x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go index db50b6929b0..4d695309775 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/quarkprovider/quarkprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go @@ -4,7 +4,7 @@ //go:build linux -package quarkprovider +package modernprovider import ( "context" From c49bfe5e5c173320045c5eb8ed95688c3e6639b7 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 18 Sep 2024 13:40:08 -0700 Subject: [PATCH 22/44] format --- x-pack/auditbeat/processors/sessionmd/add_session_metadata.go | 2 +- .../sessionmd/provider/modernprovider/modernprovider.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index bbe1b103e7f..656bc7d3670 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -17,8 +17,8 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/modernprovider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go index 4d695309775..da3d9c2661f 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go @@ -125,7 +125,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e ctx: ctx, logger: logger, qq: qq, - qqMtx: &qqMtx, + qqMtx: &qqMtx, } go func(qq *quark.Queue, logger *logp.Logger, p *prvdr) { From f3ace300255eee549c375690b4fa604a8ed32984 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 20 Sep 2024 15:08:23 -0700 Subject: [PATCH 23/44] use quark package --- .gitmodules | 3 --- go.mod | 3 +-- go.sum | 2 ++ .../sessionmd/provider/modernprovider/modernprovider.go | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 3a8bc15fbef..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "x-pack/auditbeat/processors/sessionmd/quark"] - path = x-pack/auditbeat/processors/sessionmd/quark - url = git@github.com:elastic/quark.git diff --git a/go.mod b/go.mod index 7f07e848533..10ddfc48b52 100644 --- a/go.mod +++ b/go.mod @@ -209,6 +209,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/icholy/digest v0.1.22 + github.com/mjwolf/quark v0.4.0 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 @@ -278,7 +279,6 @@ require ( github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/pkcs8 v1.0.0 // indirect - github.com/elastic/quark/go v0.0.0-00010101000000-000000000000 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fearful-symmetry/gomsr v0.0.1 // indirect @@ -405,7 +405,6 @@ replace ( github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 - github.com/elastic/quark/go => /home/mwolf/git/beats/x-pack/auditbeat/processors/sessionmd/quark/go github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c github.com/golang/glog => github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4 diff --git a/go.sum b/go.sum index df15b7c4791..d30e35ac72f 100644 --- a/go.sum +++ b/go.sum @@ -1326,6 +1326,8 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mjwolf/quark v0.4.0 h1:adKQdurcNLUMp/jdql5jp89S9TYwitOo+YKd1QAKvC4= +github.com/mjwolf/quark v0.4.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go index da3d9c2661f..92472b29fbc 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go @@ -22,7 +22,7 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" - quark "github.com/elastic/quark/go" + quark "github.com/mjwolf/quark" ) type prvdr struct { From 635b33aeada2f07fcd42df76f4e909333b840871 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 23 Sep 2024 16:21:59 -0700 Subject: [PATCH 24/44] fix lint errors --- go.mod | 2 +- go.sum | 4 +++ .../processors/sessionmd/procfs/procfs.go | 4 +-- .../provider/kprobeprovider/kprobeprovider.go | 7 +++-- .../provider/modernprovider/modernprovider.go | 29 ++++--------------- 5 files changed, 18 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 7848031f5b6..76ec5bbf5b7 100644 --- a/go.mod +++ b/go.mod @@ -212,7 +212,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/icholy/digest v0.1.22 github.com/meraki/dashboard-api-go/v3 v3.0.9 - github.com/mjwolf/quark v0.4.0 + github.com/mjwolf/quark v0.6.0 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 diff --git a/go.sum b/go.sum index 67e619be07d..f9d53f6032e 100644 --- a/go.sum +++ b/go.sum @@ -1338,6 +1338,10 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mjwolf/quark v0.4.0 h1:adKQdurcNLUMp/jdql5jp89S9TYwitOo+YKd1QAKvC4= github.com/mjwolf/quark v0.4.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= +github.com/mjwolf/quark v0.5.0 h1:+7A9t1u0Yz/EuEd6srZnZRA+LCwIaBkFjN5iN8289dc= +github.com/mjwolf/quark v0.5.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= +github.com/mjwolf/quark v0.6.0 h1:D7FIBbSMDjQ/bt/NLIGf5s95aJWS/PH73ejgEGKE2v8= +github.com/mjwolf/quark v0.6.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= diff --git a/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go index 116d4e64901..dcb1ec666bd 100644 --- a/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go +++ b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go @@ -20,11 +20,11 @@ import ( ) func MajorTTY(ttyNr uint32) uint32 { - return uint32((ttyNr >> 8) & 0xff) + return (ttyNr >> 8) & 0xff } func MinorTTY(ttyNr uint32) uint32 { - return uint32(((ttyNr >> 12) & 0xfff00) | (ttyNr & 0xff)) + return ((ttyNr >> 12) & 0xfff00) | (ttyNr & 0xff) } // this interface exists so that we can inject a mock procfs reader for deterministic testing diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go index 0219914437c..faccaf0ea54 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go @@ -42,15 +42,18 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr // Load kprobe eventsFile, err := os.OpenFile(tracingEvents, os.O_APPEND|os.O_RDWR, 0777) if err != nil { - return nil, fmt.Errorf("opening %v: %v", tracingEvents, err) + return nil, fmt.Errorf("opening %v: %w", tracingEvents, err) } defer eventsFile.Close() if _, err := eventsFile.WriteString(loadExecve); err != nil { - return nil, fmt.Errorf("loading execve kprobe: %v", err) + return nil, fmt.Errorf("loading execve kprobe: %w", err) } pipeFile, err := os.OpenFile(tracingPipe, os.O_RDONLY, 0644) + if err != nil { + return nil, fmt.Errorf("opening trace pipe: %w", err) + } //Read from trace pipe, and insert events into DB. go func(f *os.File, logger *logp.Logger) { reader := bufio.NewReader(f) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go index 92472b29fbc..4850894f8c2 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go @@ -18,11 +18,12 @@ import ( "sync" "time" + quark "github.com/mjwolf/quark" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" - quark "github.com/mjwolf/quark" ) type prvdr struct { @@ -53,24 +54,6 @@ const ( EntryUnknown EntryType = "unknown" ) -var containerRuntimes = [...]string{ - "containerd-shim", - "runc", - "conmon", -} - -// "filtered" executables are executables that relate to internal -// implementation details of entry mechanisms. The set of circumstances under -// which they can become an entry leader are reduced compared to other binaries -// (see implementation and unit tests). -var filteredExecutables = [...]string{ - "runc", - "containerd-shim", - "calico-node", - "check-status", - "conmon", -} - const ( ptsMinMajor = 136 ptsMaxMajor = 143 @@ -117,7 +100,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e attr.Flags = quark.QQ_KPROBE | quark.QQ_MIN_AGG | quark.QQ_ENTRY_LEADER qq, err := quark.OpenQueue(attr, 64) if err != nil { - return nil, fmt.Errorf("open queue: %v", err) + return nil, fmt.Errorf("open queue: %w", err) } var qqMtx sync.Mutex @@ -134,7 +117,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e procs, err := qq.GetEvents() p.qqMtx.Unlock() if err != nil { - logger.Errorf("get events from quark: %v", err) + logger.Errorf("get events from quark: %w", err) continue } for _, proc := range procs { @@ -143,7 +126,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e if len(procs) == 0 { err = qq.Block() if err != nil { - logger.Errorf("quark block: %v", err) + logger.Errorf("quark block: %w", err) continue } } @@ -280,7 +263,7 @@ func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { func (p prvdr) lookupLocked(pid uint32) (quark.Process, bool) { p.qqMtx.Lock() - p.qqMtx.Unlock() + defer p.qqMtx.Unlock() return p.qq.Lookup(int(pid)) } From 82252b1aa97e166efdd65acbbae2c4da0241a275 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 23 Sep 2024 16:34:45 -0700 Subject: [PATCH 25/44] add license override for quark --- dev-tools/notice/overrides.json | 1 + .../sessionmd/provider/modernprovider/modernprovider.go | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dev-tools/notice/overrides.json b/dev-tools/notice/overrides.json index eee18acc0de..b9150d3ba9e 100644 --- a/dev-tools/notice/overrides.json +++ b/dev-tools/notice/overrides.json @@ -18,3 +18,4 @@ {"name": "github.com/dnaeon/go-vcr", "licenceType": "BSD-2-Clause"} {"name": "github.com/JohnCGriffin/overflow", "licenceType": "MIT"} {"name": "github.com/elastic/ebpfevents", "licenceType": "Apache-2.0"} +{"name": "github.com/mjwolf/quark", "licenceType": "Apache-2.0"} diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go index 4850894f8c2..d5a2f23241a 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go @@ -66,8 +66,6 @@ const ( var ( bootID string pidNsInode uint64 - initError error - once sync.Once ) func readBootID() (string, error) { From b512748f7021405817b475945a4b7f6907fe8d1c Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 23 Sep 2024 16:57:02 -0700 Subject: [PATCH 26/44] update notice --- NOTICE.txt | 12 ++++++++++++ go.sum | 4 ---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 6c99d84597e..a2f26cf9395 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21199,6 +21199,18 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/mjwolf/quark +Version: v0.6.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/mjwolf/quark@v0.6.0/LICENSE.txt: + +Source code in this repository is licensed under the Apache License Version 2.0, +an Apache compatible license. + + -------------------------------------------------------------------------------- Dependency : github.com/olekukonko/tablewriter Version: v0.0.5 diff --git a/go.sum b/go.sum index c654946f4de..ede3099a0f4 100644 --- a/go.sum +++ b/go.sum @@ -1333,10 +1333,6 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mjwolf/quark v0.4.0 h1:adKQdurcNLUMp/jdql5jp89S9TYwitOo+YKd1QAKvC4= -github.com/mjwolf/quark v0.4.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= -github.com/mjwolf/quark v0.5.0 h1:+7A9t1u0Yz/EuEd6srZnZRA+LCwIaBkFjN5iN8289dc= -github.com/mjwolf/quark v0.5.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= github.com/mjwolf/quark v0.6.0 h1:D7FIBbSMDjQ/bt/NLIGf5s95aJWS/PH73ejgEGKE2v8= github.com/mjwolf/quark v0.6.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= From 3b4019d3813b9bd6312f9b48ee1f46b9d9a023d1 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 23 Sep 2024 17:13:09 -0700 Subject: [PATCH 27/44] update quark version --- NOTICE.txt | 208 +++++++++++++++++++++++++++++++++++++++++++++++++++-- go.mod | 2 +- go.sum | 4 +- 3 files changed, 207 insertions(+), 7 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index a2f26cf9395..78896c1fe55 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21201,14 +21201,214 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/mjwolf/quark -Version: v0.6.0 +Version: v0.7.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/mjwolf/quark@v0.6.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/mjwolf/quark@v0.7.0/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. -Source code in this repository is licensed under the Apache License Version 2.0, -an Apache compatible license. + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -------------------------------------------------------------------------------- diff --git a/go.mod b/go.mod index 50cca29a0de..4b82cc4a70b 100644 --- a/go.mod +++ b/go.mod @@ -212,7 +212,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/icholy/digest v0.1.22 github.com/meraki/dashboard-api-go/v3 v3.0.9 - github.com/mjwolf/quark v0.6.0 + github.com/mjwolf/quark v0.7.0 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 diff --git a/go.sum b/go.sum index ede3099a0f4..479377ed8fd 100644 --- a/go.sum +++ b/go.sum @@ -1333,8 +1333,8 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mjwolf/quark v0.6.0 h1:D7FIBbSMDjQ/bt/NLIGf5s95aJWS/PH73ejgEGKE2v8= -github.com/mjwolf/quark v0.6.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= +github.com/mjwolf/quark v0.7.0 h1:Q/s9M4UZ1bT/h0G0sx1bgml3zq3ay3jq+0aH5l+pO8g= +github.com/mjwolf/quark v0.7.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= From 72c5988ea566bb26ea980bcff11ee5f2cd59204a Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 24 Sep 2024 13:49:59 -0700 Subject: [PATCH 28/44] update quark ver --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 78896c1fe55..590e7f7309e 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21201,11 +21201,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/mjwolf/quark -Version: v0.7.0 +Version: v0.8.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/mjwolf/quark@v0.7.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/mjwolf/quark@v0.8.0/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 4b82cc4a70b..44f671f21d5 100644 --- a/go.mod +++ b/go.mod @@ -212,7 +212,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/icholy/digest v0.1.22 github.com/meraki/dashboard-api-go/v3 v3.0.9 - github.com/mjwolf/quark v0.7.0 + github.com/mjwolf/quark v0.8.0 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 diff --git a/go.sum b/go.sum index 479377ed8fd..c7305ae1754 100644 --- a/go.sum +++ b/go.sum @@ -1333,8 +1333,8 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mjwolf/quark v0.7.0 h1:Q/s9M4UZ1bT/h0G0sx1bgml3zq3ay3jq+0aH5l+pO8g= -github.com/mjwolf/quark v0.7.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= +github.com/mjwolf/quark v0.8.0 h1:fQshQ/IhWgJoeMicJC2BDzloEIcfmBim67AJc9a9+AE= +github.com/mjwolf/quark v0.8.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= From b4e79bf2b8879b4f71b46962bd140104a19f4922 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 7 Oct 2024 15:32:50 -0700 Subject: [PATCH 29/44] use go-quark in sessionmd --- go.mod | 7 ++--- go.sum | 26 +++++-------------- .../provider/modernprovider/modernprovider.go | 2 +- x-pack/auditbeat/processors/sessionmd/quark | 1 - 4 files changed, 9 insertions(+), 27 deletions(-) delete mode 160000 x-pack/auditbeat/processors/sessionmd/quark diff --git a/go.mod b/go.mod index 44f671f21d5..75fb29cb6c0 100644 --- a/go.mod +++ b/go.mod @@ -135,7 +135,7 @@ require ( github.com/ugorji/go/codec v1.1.8 github.com/vmware/govmomi v0.39.0 go.elastic.co/ecszap v1.0.2 - go.elastic.co/go-licence-detector v0.6.1 + go.elastic.co/go-licence-detector v0.7.0 go.etcd.io/bbolt v1.3.10 go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 @@ -212,7 +212,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/icholy/digest v0.1.22 github.com/meraki/dashboard-api-go/v3 v3.0.9 - github.com/mjwolf/quark v0.8.0 + github.com/mjwolf/go-quark v0.1.3 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 @@ -298,7 +298,6 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/go-resty/resty/v2 v2.11.0 // indirect github.com/go-stack/stack v1.8.0 // indirect - github.com/gobuffalo/here v0.6.7 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/godror/knownpb v0.1.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -327,7 +326,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/karrick/godirwalk v1.17.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.9 // indirect @@ -336,7 +334,6 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/markbates/pkger v0.17.1 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect diff --git a/go.sum b/go.sum index c7305ae1754..0b4726f60cc 100644 --- a/go.sum +++ b/go.sum @@ -839,9 +839,6 @@ github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= -github.com/gobuffalo/here v0.6.7 h1:hpfhh+kt2y9JLDfhYUxxCRxQol540jsVfKUZzjlbp8o= -github.com/gobuffalo/here v0.6.7/go.mod h1:vuCfanjqckTuRlqAitJz6QC4ABNnS27wLb816UhsPcc= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= @@ -969,7 +966,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= github.com/google/licenseclassifier v0.0.0-20221004142553-c1ed8fcf4bab h1:okY7fFoWybMbxiHkaqStN4mxSrPfYmTZl5Zh32Z5FjY= github.com/google/licenseclassifier v0.0.0-20221004142553-c1ed8fcf4bab/go.mod h1:jkYIPv59uiw+1MxTWlqQEKebsUDV1DCXQtBBn5lVzf4= github.com/google/licenseclassifier/v2 v2.0.0-alpha.1/go.mod h1:YAgBGGTeNDMU+WfIgaFvjZe4rudym4f6nIn8ZH5X+VM= @@ -1199,9 +1195,6 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= -github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -1267,9 +1260,6 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/pkger v0.17.0/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= -github.com/markbates/pkger v0.17.1 h1:/MKEtWqtc0mZvu9OinB9UzVN9iYCwLWuyUv4Bw+PCno= -github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 h1:YFh+sjyJTMQSYjKwM4dFKhJPJC/wfo98tPUc17HdoYw= github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI= @@ -1333,8 +1323,10 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mjwolf/quark v0.8.0 h1:fQshQ/IhWgJoeMicJC2BDzloEIcfmBim67AJc9a9+AE= -github.com/mjwolf/quark v0.8.0/go.mod h1:o3qk32z0U4JrIUUik5b6+LZXjuh2zkeoP9mvZYLDiW0= +github.com/mjwolf/go-quark v0.1.2 h1:HPU2y/zDI3cemTLfFPyQ0xyI3w+m7tcumOldVFt98vU= +github.com/mjwolf/go-quark v0.1.2/go.mod h1:qHZFSWAGUg6eOpvideVWvTtWx/HKvdECdb1d3PrLBow= +github.com/mjwolf/go-quark v0.1.3 h1:3dmrH/mP10YuTlZJ9TtdmyGLaLhzZl0V1pG3qHAK3Mg= +github.com/mjwolf/go-quark v0.1.3/go.mod h1:qHZFSWAGUg6eOpvideVWvTtWx/HKvdECdb1d3PrLBow= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= @@ -1710,8 +1702,8 @@ go.elastic.co/ecszap v1.0.2 h1:iW5OGx8IiokiUzx/shD4AJCPFMC9uUtr7ycaiEIU++I= go.elastic.co/ecszap v1.0.2/go.mod h1:dJkSlK3BTiwG/qXhCwe50Mz/jwu854vSip8sIeQhNZg= go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= -go.elastic.co/go-licence-detector v0.6.1 h1:T2PFHYdow+9mAjj6K5ehn5anTxtsURfom2P4S6PgMzg= -go.elastic.co/go-licence-detector v0.6.1/go.mod h1:qQ1clBRS2f0Ee5ie+y2LLYnyhSNJNm0Ha6d7SoYVtM4= +go.elastic.co/go-licence-detector v0.7.0 h1:qC31sfyfNcNx/zMYcLABU0ac3MbGHZgksCAb5lMDUMg= +go.elastic.co/go-licence-detector v0.7.0/go.mod h1:f5ty8pjynzQD8BcS+s0qtlOGKc35/HKQxCVi8SHhV5k= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= @@ -1892,8 +1884,6 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1970,7 +1960,6 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= @@ -2006,8 +1995,6 @@ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2257,7 +2244,6 @@ golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go index d5a2f23241a..cea96fe13e3 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go @@ -18,7 +18,7 @@ import ( "sync" "time" - quark "github.com/mjwolf/quark" + quark "github.com/mjwolf/go-quark" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" diff --git a/x-pack/auditbeat/processors/sessionmd/quark b/x-pack/auditbeat/processors/sessionmd/quark deleted file mode 160000 index 72525ccc03c..00000000000 --- a/x-pack/auditbeat/processors/sessionmd/quark +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 72525ccc03c9b0ceeeae6aa6c65994a2f12c5517 From d485cccdf215c88c9f25ff770177c93b1bd96ea7 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 7 Oct 2024 17:07:03 -0700 Subject: [PATCH 30/44] fix lint error --- .../provider/kprobeprovider/kprobeprovider.go | 77 ------------------- .../provider/procfsprovider/procfsprovider.go | 6 +- 2 files changed, 5 insertions(+), 78 deletions(-) delete mode 100644 x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go deleted file mode 100644 index faccaf0ea54..00000000000 --- a/x-pack/auditbeat/processors/sessionmd/provider/kprobeprovider/kprobeprovider.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build linux - -package kprobeprovider - -import ( - "bufio" - "context" - "fmt" - "os" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" - "github.com/elastic/elastic-agent-libs/logp" -) - -const ( - tracingEvents = "/sys/kernel/debug/tracing/kprobe_events" - tracingPipe = "/sys/kernel/tracing/trace_pipe" - - loadExecve = "p:kprobes/my_probe sys_execve\n" -) - -type prvdr struct { - ctx context.Context - logger *logp.Logger - db *processdb.DB -} - -func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { - p := prvdr{ - ctx: ctx, - logger: logger, - db: db, - } - - // Load kprobe - eventsFile, err := os.OpenFile(tracingEvents, os.O_APPEND|os.O_RDWR, 0777) - if err != nil { - return nil, fmt.Errorf("opening %v: %w", tracingEvents, err) - } - defer eventsFile.Close() - - if _, err := eventsFile.WriteString(loadExecve); err != nil { - return nil, fmt.Errorf("loading execve kprobe: %w", err) - } - - pipeFile, err := os.OpenFile(tracingPipe, os.O_RDONLY, 0644) - if err != nil { - return nil, fmt.Errorf("opening trace pipe: %w", err) - } - //Read from trace pipe, and insert events into DB. - go func(f *os.File, logger *logp.Logger) { - reader := bufio.NewReader(f) - for { - line, err := reader.ReadString('\n') - if err != nil { - logger.Errorf("reading event pipe: %v", err) - } - logger.Errorf("MWOLF: pipe: %v", line) - } - }(pipeFile, logger) - return &p, nil -} - -func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { - return nil, fmt.Errorf("not implemented") -} - -func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { - return nil -} diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index af1b481c4e2..4d050ace2c5 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -91,7 +91,11 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { if err != nil { goto out } - pe.CWD = intr.(string) + if str, ok := intr.(string); ok { + pe.CWD = str + } else { + goto out + } out: } p.db.InsertExec(pe) From 84cc43669d2b8d2899ab12e39a598ef239f5f759 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 8 Oct 2024 10:30:30 -0700 Subject: [PATCH 31/44] Only use modernprovider on build that are supported by go-quark --- ...ernprovider.go => modernprovider_linux.go} | 2 +- .../modernprovider/modernprovider_other.go | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) rename x-pack/auditbeat/processors/sessionmd/provider/modernprovider/{modernprovider.go => modernprovider_linux.go} (99%) create mode 100644 x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_other.go diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go similarity index 99% rename from x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go rename to x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go index cea96fe13e3..e781334401f 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -//go:build linux +//go:build linux && (amd64 || arm64) && cgo package modernprovider diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_other.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_other.go new file mode 100644 index 00000000000..f0d3bd75dca --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_other.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !(linux && (amd64 || arm64) && cgo) + +package modernprovider + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + "github.com/elastic/elastic-agent-libs/logp" +) + +type prvdr struct{} + +func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, error) { + return prvdr{}, fmt.Errorf("build type not supported, cgo required") +} + +func (p prvdr) SyncDB(event *beat.Event, pid uint32) error { + return fmt.Errorf("build type not supported") +} + +func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { + return nil, fmt.Errorf("build type not supported") +} From 491090a3f53ba3c44c8ec0e6074e16c7875e785a Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 9 Oct 2024 05:50:08 -0700 Subject: [PATCH 32/44] Update go-quark --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e7e08f48d2b..67556f16c18 100644 --- a/go.mod +++ b/go.mod @@ -207,7 +207,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/icholy/digest v0.1.22 github.com/meraki/dashboard-api-go/v3 v3.0.9 - github.com/mjwolf/go-quark v0.1.3 + github.com/mjwolf/go-quark v0.2.0 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 diff --git a/go.sum b/go.sum index dc1066cbae8..f3c637f2e60 100644 --- a/go.sum +++ b/go.sum @@ -719,6 +719,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mjwolf/go-quark v0.1.3 h1:3dmrH/mP10YuTlZJ9TtdmyGLaLhzZl0V1pG3qHAK3Mg= github.com/mjwolf/go-quark v0.1.3/go.mod h1:qHZFSWAGUg6eOpvideVWvTtWx/HKvdECdb1d3PrLBow= +github.com/mjwolf/go-quark v0.2.0 h1:arAKLySXUzueSbaUkvSo7CJQrIicWorbSQ7P9I0MxdY= +github.com/mjwolf/go-quark v0.2.0/go.mod h1:qHZFSWAGUg6eOpvideVWvTtWx/HKvdECdb1d3PrLBow= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= From 37c3ce2eb32806e549f8a6a6f0d32866466bc3da Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 9 Oct 2024 06:10:21 -0700 Subject: [PATCH 33/44] update notice --- NOTICE.txt | 4 ++-- go.sum | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 0fa6ed518e0..5475c6dac3c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21168,11 +21168,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/mjwolf/go-quark -Version: v0.1.3 +Version: v0.2.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/mjwolf/go-quark@v0.1.3/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/mjwolf/go-quark@v0.2.0/LICENSE.txt: Source code in this repository is licensed under the Apache License Version 2.0, an Apache compatible license. diff --git a/go.sum b/go.sum index f3c637f2e60..ffc8c8d9c0a 100644 --- a/go.sum +++ b/go.sum @@ -717,8 +717,6 @@ github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWe github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mjwolf/go-quark v0.1.3 h1:3dmrH/mP10YuTlZJ9TtdmyGLaLhzZl0V1pG3qHAK3Mg= -github.com/mjwolf/go-quark v0.1.3/go.mod h1:qHZFSWAGUg6eOpvideVWvTtWx/HKvdECdb1d3PrLBow= github.com/mjwolf/go-quark v0.2.0 h1:arAKLySXUzueSbaUkvSo7CJQrIicWorbSQ7P9I0MxdY= github.com/mjwolf/go-quark v0.2.0/go.mod h1:qHZFSWAGUg6eOpvideVWvTtWx/HKvdECdb1d3PrLBow= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= From aa10e60371c784c7431acdb6b81b0dadac19a0c9 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 9 Oct 2024 19:04:57 -0700 Subject: [PATCH 34/44] revert seccomp change --- x-pack/auditbeat/processors/sessionmd/types/process.go | 1 - x-pack/auditbeat/seccomp_linux.go | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/types/process.go b/x-pack/auditbeat/processors/sessionmd/types/process.go index ee05206636d..a437f35310f 100644 --- a/x-pack/auditbeat/processors/sessionmd/types/process.go +++ b/x-pack/auditbeat/processors/sessionmd/types/process.go @@ -451,7 +451,6 @@ func (p *Process) ToMap() mapstr.M { if p.End != nil { process.Put("end", p.End) } - // TODO: are other Ends needed, ancestors shouldn't end before process return process } diff --git a/x-pack/auditbeat/seccomp_linux.go b/x-pack/auditbeat/seccomp_linux.go index 00184b409e0..4eb0529def0 100644 --- a/x-pack/auditbeat/seccomp_linux.go +++ b/x-pack/auditbeat/seccomp_linux.go @@ -27,7 +27,6 @@ func init() { // The system/socket dataset uses additional syscalls if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, - "eventfd2", "mount", "mq_open", // required for creds kprobe guess trigger. "perf_event_open", From 84a216f4f51a80a5a866f8384b43f9044e6731c2 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 10 Oct 2024 13:37:08 -0700 Subject: [PATCH 35/44] revert seccomp changes --- x-pack/auditbeat/seccomp_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/seccomp_linux.go b/x-pack/auditbeat/seccomp_linux.go index 4eb0529def0..709d973465d 100644 --- a/x-pack/auditbeat/seccomp_linux.go +++ b/x-pack/auditbeat/seccomp_linux.go @@ -17,7 +17,6 @@ func init() { // requirements beyond the default policy from libbeat so whitelist // these additional syscalls. if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, - "faccessat2", "mremap", "umask", "setreuid", @@ -27,6 +26,7 @@ func init() { // The system/socket dataset uses additional syscalls if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, + "eventfd2", "mount", "mq_open", // required for creds kprobe guess trigger. "perf_event_open", From 38901767a9c797a685be6d91b95cda1331f6539e Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 11 Oct 2024 09:18:10 -0700 Subject: [PATCH 36/44] Use replace directive in go.mod --- NOTICE.txt | 24 +++++++++---------- go.mod | 4 ++-- .../modernprovider/modernprovider_linux.go | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index fead5ec6865..8b21c603cde 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -14745,6 +14745,18 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/mjwolf/go-quark +Version: v0.2.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/mjwolf/go-quark@v0.2.0/LICENSE.txt: + +Source code in this repository is licensed under the Apache License Version 2.0, +an Apache compatible license. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-seccomp-bpf Version: v1.4.0 @@ -21166,18 +21178,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/mjwolf/go-quark -Version: v0.2.0 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mjwolf/go-quark@v0.2.0/LICENSE.txt: - -Source code in this repository is licensed under the Apache License Version 2.0, -an Apache compatible license. - - -------------------------------------------------------------------------------- Dependency : github.com/olekukonko/tablewriter Version: v0.0.5 diff --git a/go.mod b/go.mod index 036fefd00c9..01f9bd3c061 100644 --- a/go.mod +++ b/go.mod @@ -192,6 +192,7 @@ require ( github.com/elastic/elastic-agent-libs v0.12.1 github.com/elastic/elastic-agent-system-metrics v0.11.1 github.com/elastic/go-elasticsearch/v8 v8.14.0 + github.com/elastic/go-quark v0.2.0 github.com/elastic/go-sfdc v0.0.0-20240621062639-bcc8456508ff github.com/elastic/mito v1.15.0 github.com/elastic/tk-btf v0.1.0 @@ -207,7 +208,6 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/icholy/digest v0.1.22 github.com/meraki/dashboard-api-go/v3 v3.0.9 - github.com/mjwolf/go-quark v0.2.0 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 @@ -412,7 +412,7 @@ replace ( github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 - + github.com/elastic/go-quark => github.com/mjwolf/go-quark v0.2.0 github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/elastic/fsnotify v1.6.1-0.20240920222514-49f82bdbc9e3 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go index e781334401f..4c235e1f5f1 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go @@ -18,7 +18,7 @@ import ( "sync" "time" - quark "github.com/mjwolf/go-quark" + quark "github.com/elastic/go-quark" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" From 08299c2adadef3e1c8eb41eb97c80fa6f4c06f36 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 14 Oct 2024 21:49:23 -0700 Subject: [PATCH 37/44] Update quark flags --- .../provider/modernprovider/modernprovider_linux.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go index 4c235e1f5f1..9050e59cf15 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go @@ -95,7 +95,7 @@ func readPIDNsInode() (uint64, error) { func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, error) { attr := quark.DefaultQueueAttr() - attr.Flags = quark.QQ_KPROBE | quark.QQ_MIN_AGG | quark.QQ_ENTRY_LEADER + attr.Flags = quark.QQ_NO_SNAPSHOT | quark.QQ_ENTRY_LEADER qq, err := quark.OpenQueue(attr, 64) if err != nil { return nil, fmt.Errorf("open queue: %w", err) @@ -112,16 +112,16 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e go func(qq *quark.Queue, logger *logp.Logger, p *prvdr) { for { p.qqMtx.Lock() - procs, err := qq.GetEvents() + events, err := qq.GetEvents() p.qqMtx.Unlock() if err != nil { logger.Errorf("get events from quark: %w", err) continue } - for _, proc := range procs { - logger.Infof("proc: %v", proc) + for _, event := range events { + logger.Infof("event: %v", event) } - if len(procs) == 0 { + if len(events) == 0 { err = qq.Block() if err != nil { logger.Errorf("quark block: %w", err) From 2da7e36fda5a6d2fe00d65fe0dc28e44dee7e7b8 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 14 Oct 2024 22:11:02 -0700 Subject: [PATCH 38/44] Use published elastic/go-quark --- NOTICE.txt | 6 +++--- dev-tools/notice/overrides.json | 2 +- go.mod | 3 +-- go.sum | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 6df6e84a281..51159b21337 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -14746,12 +14746,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/mjwolf/go-quark -Version: v0.2.0 +Dependency : github.com/elastic/go-quark +Version: v0.1.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/mjwolf/go-quark@v0.2.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-quark@v0.1.0/LICENSE.txt: Source code in this repository is licensed under the Apache License Version 2.0, an Apache compatible license. diff --git a/dev-tools/notice/overrides.json b/dev-tools/notice/overrides.json index d5eff423ccf..a50cac02e0f 100644 --- a/dev-tools/notice/overrides.json +++ b/dev-tools/notice/overrides.json @@ -19,4 +19,4 @@ {"name": "github.com/JohnCGriffin/overflow", "licenceType": "MIT"} {"name": "github.com/elastic/ebpfevents", "licenceType": "Apache-2.0"} {"name": "go.opentelemetry.io/collector/config/configopaque", "licenceType": "Apache-2.0"} -{"name": "github.com/mjwolf/go-quark", "licenceType": "Apache-2.0"} +{"name": "github.com/elastic/go-quark", "licenceType": "Apache-2.0"} diff --git a/go.mod b/go.mod index 3f3d0789037..dc3befe6df6 100644 --- a/go.mod +++ b/go.mod @@ -192,7 +192,7 @@ require ( github.com/elastic/elastic-agent-libs v0.12.1 github.com/elastic/elastic-agent-system-metrics v0.11.1 github.com/elastic/go-elasticsearch/v8 v8.14.0 - github.com/elastic/go-quark v0.2.0 + github.com/elastic/go-quark v0.1.0 github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 github.com/elastic/mito v1.15.0 github.com/elastic/tk-btf v0.1.0 @@ -425,7 +425,6 @@ replace ( github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details. github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 - github.com/elastic/go-quark => github.com/mjwolf/go-quark v0.2.0 github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/elastic/fsnotify v1.6.1-0.20240920222514-49f82bdbc9e3 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20241002174017-e8c5fda595e6 diff --git a/go.sum b/go.sum index b8d4c263bbf..0d85e961cbe 100644 --- a/go.sum +++ b/go.sum @@ -381,6 +381,8 @@ github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 h1:q8n4QjcLa4q39Q3fqHRknTBXBtegjriHFrB42YKgXGI= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595/go.mod h1:s09U1b4P1ZxnKx2OsqY7KlHdCesqZWIhyq0Gs/QC/Us= +github.com/elastic/go-quark v0.1.0 h1:VYjlWGlbjzcg6IqdqlykPw0rg2Za7cFnb9hDSoqs9MQ= +github.com/elastic/go-quark v0.1.0/go.mod h1:/ngqgumD/Z5vnFZ4XPN2kCbxnEfG5/Uc+bRvOBabVVA= github.com/elastic/go-seccomp-bpf v1.4.0 h1:6y3lYrEHrLH9QzUgOiK8WDqmPaMnnB785WxibCNIOH4= github.com/elastic/go-seccomp-bpf v1.4.0/go.mod h1:wIMxjTbKpWGQk4CV9WltlG6haB4brjSH/dvAohBPM1I= github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 h1:yuiN60oaQUz2PtNpNhDI2H6zrCdfiiptmNdwV5WUaKA= @@ -739,8 +741,6 @@ github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWe github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mjwolf/go-quark v0.2.0 h1:arAKLySXUzueSbaUkvSo7CJQrIicWorbSQ7P9I0MxdY= -github.com/mjwolf/go-quark v0.2.0/go.mod h1:qHZFSWAGUg6eOpvideVWvTtWx/HKvdECdb1d3PrLBow= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= From 48c510db00feb702b31aab1230078038dd0e96bb Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 15 Oct 2024 09:49:29 -0700 Subject: [PATCH 39/44] upgrade to go-quark v0.1.1 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- .../provider/modernprovider/modernprovider_linux.go | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 925c67c5069..42e9fca7279 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -14747,11 +14747,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-quark -Version: v0.1.0 +Version: v0.1.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-quark@v0.1.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-quark@v0.1.1/LICENSE.txt: Source code in this repository is licensed under the Apache License Version 2.0, an Apache compatible license. diff --git a/go.mod b/go.mod index d59585907fc..410d0e17c65 100644 --- a/go.mod +++ b/go.mod @@ -192,7 +192,7 @@ require ( github.com/elastic/elastic-agent-libs v0.12.1 github.com/elastic/elastic-agent-system-metrics v0.11.1 github.com/elastic/go-elasticsearch/v8 v8.14.0 - github.com/elastic/go-quark v0.1.0 + github.com/elastic/go-quark v0.1.1 github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 github.com/elastic/mito v1.15.0 github.com/elastic/tk-btf v0.1.0 diff --git a/go.sum b/go.sum index b5198a97395..34d47850c8d 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 h1:q8n4QjcLa4q39Q3fqHRknTBXBtegjriHFrB42YKgXGI= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595/go.mod h1:s09U1b4P1ZxnKx2OsqY7KlHdCesqZWIhyq0Gs/QC/Us= -github.com/elastic/go-quark v0.1.0 h1:VYjlWGlbjzcg6IqdqlykPw0rg2Za7cFnb9hDSoqs9MQ= -github.com/elastic/go-quark v0.1.0/go.mod h1:/ngqgumD/Z5vnFZ4XPN2kCbxnEfG5/Uc+bRvOBabVVA= +github.com/elastic/go-quark v0.1.1 h1:0hR8dspVKzR9ZdpEQHnpF1AQ5S6bGuJ8rs/WOVrWhSU= +github.com/elastic/go-quark v0.1.1/go.mod h1:/ngqgumD/Z5vnFZ4XPN2kCbxnEfG5/Uc+bRvOBabVVA= github.com/elastic/go-seccomp-bpf v1.4.0 h1:6y3lYrEHrLH9QzUgOiK8WDqmPaMnnB785WxibCNIOH4= github.com/elastic/go-seccomp-bpf v1.4.0/go.mod h1:wIMxjTbKpWGQk4CV9WltlG6haB4brjSH/dvAohBPM1I= github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 h1:yuiN60oaQUz2PtNpNhDI2H6zrCdfiiptmNdwV5WUaKA= diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go index 9050e59cf15..e7849f20818 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go @@ -504,17 +504,17 @@ func getGroupName(id string) (string, bool) { func getEntryTypeName(entryType uint32) string { switch int(entryType) { case quark.QUARK_ELT_INIT: - return "init" + return string(Init) case quark.QUARK_ELT_SSHD: - return "sshd" + return string(Sshd) case quark.QUARK_ELT_SSM: - return "ssm" + return string(Ssm) case quark.QUARK_ELT_CONTAINER: - return "container" + return string(Container) case quark.QUARK_ELT_TERM: - return "terminal" + return string(Terminal) case quark.QUARK_ELT_CONSOLE: - return "console" + return string(EntryConsole) case quark.QUARK_ELT_KTHREAD: return "kthread" default: From 07b8576fe0884aef24ec6d51dc736db51ec2505f Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 15 Oct 2024 09:53:03 -0700 Subject: [PATCH 40/44] stop enrichment on error --- .../sessionmd/provider/modernprovider/modernprovider_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go index e7849f20818..7f8cde2144b 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go @@ -116,7 +116,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e p.qqMtx.Unlock() if err != nil { logger.Errorf("get events from quark: %w", err) - continue + break } for _, event := range events { logger.Infof("event: %v", event) @@ -125,7 +125,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e err = qq.Block() if err != nil { logger.Errorf("quark block: %w", err) - continue + break } } } From 71e9d2790abb7ee45c355ae784c0c2b5a550e023 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 15 Oct 2024 11:40:28 -0700 Subject: [PATCH 41/44] update to go-quark v0.1.2 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 42e9fca7279..3c496f2cdd6 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -14747,11 +14747,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-quark -Version: v0.1.1 +Version: v0.1.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-quark@v0.1.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-quark@v0.1.2/LICENSE.txt: Source code in this repository is licensed under the Apache License Version 2.0, an Apache compatible license. diff --git a/go.mod b/go.mod index 410d0e17c65..23da592308f 100644 --- a/go.mod +++ b/go.mod @@ -192,7 +192,7 @@ require ( github.com/elastic/elastic-agent-libs v0.12.1 github.com/elastic/elastic-agent-system-metrics v0.11.1 github.com/elastic/go-elasticsearch/v8 v8.14.0 - github.com/elastic/go-quark v0.1.1 + github.com/elastic/go-quark v0.1.2 github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 github.com/elastic/mito v1.15.0 github.com/elastic/tk-btf v0.1.0 diff --git a/go.sum b/go.sum index 34d47850c8d..4567fa045d5 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 h1:q8n4QjcLa4q39Q3fqHRknTBXBtegjriHFrB42YKgXGI= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595/go.mod h1:s09U1b4P1ZxnKx2OsqY7KlHdCesqZWIhyq0Gs/QC/Us= -github.com/elastic/go-quark v0.1.1 h1:0hR8dspVKzR9ZdpEQHnpF1AQ5S6bGuJ8rs/WOVrWhSU= -github.com/elastic/go-quark v0.1.1/go.mod h1:/ngqgumD/Z5vnFZ4XPN2kCbxnEfG5/Uc+bRvOBabVVA= +github.com/elastic/go-quark v0.1.2 h1:Hnov9q8D9ofS976SODWWYAZ23IpgPILxTUCiccmhw0c= +github.com/elastic/go-quark v0.1.2/go.mod h1:/ngqgumD/Z5vnFZ4XPN2kCbxnEfG5/Uc+bRvOBabVVA= github.com/elastic/go-seccomp-bpf v1.4.0 h1:6y3lYrEHrLH9QzUgOiK8WDqmPaMnnB785WxibCNIOH4= github.com/elastic/go-seccomp-bpf v1.4.0/go.mod h1:wIMxjTbKpWGQk4CV9WltlG6haB4brjSH/dvAohBPM1I= github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 h1:yuiN60oaQUz2PtNpNhDI2H6zrCdfiiptmNdwV5WUaKA= From 87a33cf26fca1b5aaedf52c73d7b05ad5853c853 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 15 Oct 2024 17:55:19 -0700 Subject: [PATCH 42/44] rework from code review feedback --- .../sessionmd/add_session_metadata.go | 30 ++-- .../sessionmd/add_session_metadata_test.go | 2 +- .../docs/add_session_metadata.asciidoc | 9 +- .../sessionmd/processdb/entry_leader_test.go | 2 +- .../kerneltracingprovider_linux.go} | 141 +++++++++--------- .../kerneltracingprovider_other.go} | 2 +- .../provider/procfsprovider/procfsprovider.go | 18 +-- 7 files changed, 108 insertions(+), 96 deletions(-) rename x-pack/auditbeat/processors/sessionmd/provider/{modernprovider/modernprovider_linux.go => kerneltracingprovider/kerneltracingprovider_linux.go} (82%) rename x-pack/auditbeat/processors/sessionmd/provider/{modernprovider/modernprovider_other.go => kerneltracingprovider/kerneltracingprovider_other.go} (97%) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 656bc7d3670..4e08c1b8fc0 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -17,7 +17,7 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/modernprovider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" cfg "github.com/elastic/elastic-agent-libs/config" @@ -36,6 +36,8 @@ func InitializeModule() { } type addSessionMetadata struct { + ctx context.Context + cancel context.CancelFunc config config logger *logp.Logger db *processdb.DB @@ -51,14 +53,15 @@ func New(cfg *cfg.C) (beat.Processor, error) { logger := logp.NewLogger(logName) - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) reader := procfs.NewProcfsReader(*logger) db, err := processdb.NewDB(reader, *logger) if err != nil { + cancel() return nil, fmt.Errorf("failed to create DB: %w", err) } - if c.Backend != "modern" { + if c.Backend != "kernel_tracing" { backfilledPIDs := db.ScrapeProcfs() logger.Infof("backfilled %d processes", len(backfilledPIDs)) } @@ -67,31 +70,37 @@ func New(cfg *cfg.C) (beat.Processor, error) { switch c.Backend { case "auto": - p, err = modernprovider.NewProvider(ctx, logger) + p, err = kerneltracingprovider.NewProvider(ctx, logger) if err != nil { // Most likely cause of error is not supporting ebpf or kprobes on system, try procfs p, err = procfsprovider.NewProvider(ctx, logger, db, reader, c.PIDField) if err != nil { + cancel() return nil, fmt.Errorf("failed to create provider: %w", err) } logger.Info("backend=auto using procfs") } else { - logger.Info("backend=auto using modern") + logger.Info("backend=auto using kernel_tracing") } case "procfs": p, err = procfsprovider.NewProvider(ctx, logger, db, reader, c.PIDField) if err != nil { + cancel() return nil, fmt.Errorf("failed to create procfs provider: %w", err) } - case "modern": - p, err = modernprovider.NewProvider(ctx, logger) + case "kernel_tracing": + p, err = kerneltracingprovider.NewProvider(ctx, logger) if err != nil { - return nil, fmt.Errorf("failed to create modern provider: %w", err) + cancel() + return nil, fmt.Errorf("failed to create kernel_tracing provider: %w", err) } default: + cancel() return nil, fmt.Errorf("unknown backend configuration") } return &addSessionMetadata{ + ctx: ctx, + cancel: cancel, config: c, logger: logger, db: db, @@ -132,6 +141,7 @@ func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { func (p *addSessionMetadata) Close() error { p.db.Close() + p.cancel() return nil } @@ -151,8 +161,8 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { } var fullProcess types.Process - if p.backend == "modern" { - // modern doesn't enrich with the processor DB; process info is taken directly from modern cache + if p.backend == "kernel_tracing" { + // kernel_tracing doesn't enrich with the processor DB; process info is taken directly from quark cache proc, err := p.provider.GetProcess(pid) if err != nil { e := fmt.Errorf("pid %v not found in db: %w", pid, err) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go index 95892482f80..a993737611b 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go @@ -361,7 +361,7 @@ func TestEnrich(t *testing.T) { require.Nil(t, err, "%s: enrich error: %w", tt.testName, err) require.NotNil(t, actual, "%s: returned nil event", tt.testName) - //Validate output + // Validate output if diff := cmp.Diff(tt.expected.Fields, actual.Fields, ignoreMissingFrom(tt.expected.Fields)); diff != "" { t.Errorf("field mismatch:\n%s", diff) } diff --git a/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc b/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc index 8f79227c229..aaddde322c1 100644 --- a/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc +++ b/x-pack/auditbeat/processors/sessionmd/docs/add_session_metadata.asciidoc @@ -40,9 +40,10 @@ This enhanced data enables the powerful {security-guide}/session-view.html[Sessi The `add_session_metadata` processor operates using various backend options. * `auto` is the recommended setting. - It attempts to use `modern` first, falling back to `procfs` if necessary, ensuring compatibility even on systems without `modern` support. -* `modern` collects process information with eBPF or kprobes. - This backend requires a system with Linux kernel 5.10.16 or above, kernel support for eBPF enabled, and auditbeat running as superuser. //TODO: ***Update requirements*** + It attempts to use `kernel_tracing` first, falling back to `procfs` if necessary, ensuring compatibility even on systems without `kernel_tracing` support. +* `kernel_tracing` collects process information with eBPF or kprobes. + This backend will prefer to use eBPF, if eBPF is not supported kprobes will be used. eBPF requires a system with Linux kernel 5.10.16 or above, kernel support for eBPF enabled, and auditbeat running as superuser. + Kprobe support required Linux kernel 3.10.0 or above, and auditbeat running as a superuser. * `procfs` collects process information with the proc filesystem. This is compatible with older systems that may not support ebpf. To gather complete process info, auditbeat requires permissions to read all process data in procfs; for example, run as a superuser or have the `SYS_PTRACE` capability. @@ -50,7 +51,7 @@ The `add_session_metadata` processor operates using various backend options. [[add-session-metadata-containers]] ===== Containers If you are running {auditbeat} in a container, the container must run in the host's PID namespace. -With the `auto` or `modern` backend, these host directories must also be mounted to the same path within the container: `/sys/kernel/debug`, `/sys/fs/bpf`. +With the `auto` or `kernel_tracing` backend, these host directories must also be mounted to the same path within the container: `/sys/kernel/debug`, `/sys/fs/bpf`. [[add-session-metadata-enable]] ==== Enable and configure Session View in {auditbeat} diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go index 74140f47f6c..fa0bc6e1799 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go @@ -1491,7 +1491,7 @@ func TestPIDReuseNewSession(t *testing.T) { ExitCode: 0, }) - //2nd session + // 2nd session x1 := bashPID x2 := sshd0PID sshd0PID = command0PID diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go similarity index 82% rename from x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go rename to x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go index 7f8cde2144b..3c2a9f9b79f 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_linux.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go @@ -4,7 +4,7 @@ //go:build linux && (amd64 || arm64) && cgo -package modernprovider +package kerneltracingprovider import ( "context" @@ -12,7 +12,7 @@ import ( "fmt" "os" "os/user" - "path" + "path/filepath" "strconv" "strings" "sync" @@ -27,10 +27,16 @@ import ( ) type prvdr struct { - ctx context.Context - logger *logp.Logger - qq *quark.Queue - qqMtx *sync.Mutex + ctx context.Context + logger *logp.Logger + qq *quark.Queue + qqMtx *sync.Mutex + combinedWait time.Duration + inBackoff bool + backoffStart time.Time + since time.Time + backoffSkipped int + mtx *sync.Mutex } type TTYType int @@ -42,16 +48,15 @@ const ( TTYConsole ) -type EntryType string - const ( - Init EntryType = "init" - Sshd EntryType = "sshd" - Ssm EntryType = "ssm" - Container EntryType = "container" - Terminal EntryType = "terminal" - EntryConsole EntryType = "console" - EntryUnknown EntryType = "unknown" + Init = "init" + Sshd = "sshd" + Ssm = "ssm" + Container = "container" + Terminal = "terminal" + Kthread = "kthread" + EntryConsole = "console" + EntryUnknown = "unknown" ) const ( @@ -60,7 +65,6 @@ const ( ttyMajor = 4 consoleMaxMinor = 63 ttyMaxMinor = 255 - retryCount = 2 ) var ( @@ -71,7 +75,7 @@ var ( func readBootID() (string, error) { bootID, err := os.ReadFile("/proc/sys/kernel/random/boot_id") if err != nil { - panic(fmt.Sprintf("could not read /proc/sys/kernel/random/boot_id: %v", err)) + return "", fmt.Errorf("could not read /proc/sys/kernel/random/boot_id, process entity IDs will not be correct: %w", err) } return strings.TrimRight(string(bootID), "\n"), nil @@ -82,54 +86,58 @@ func readPIDNsInode() (uint64, error) { pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") if err != nil { - panic(fmt.Sprintf("could not read /proc/self/ns/pid: %v", err)) + return 0, fmt.Errorf("could not read /proc/self/ns/pid, process entity IDs will not be correct: %w", err) } if _, err = fmt.Sscanf(pidNsInodeRaw, "pid:[%d]", &ret); err != nil { - panic(fmt.Sprintf("could not parse contents of /proc/self/ns/pid (%s): %v", pidNsInodeRaw, err)) + return 0, fmt.Errorf("could not parse contents of /proc/self/ns/pid (%s)process entity IDs will not be correct: %w", pidNsInodeRaw, err) } return ret, nil } func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, error) { - attr := quark.DefaultQueueAttr() - attr.Flags = quark.QQ_NO_SNAPSHOT | quark.QQ_ENTRY_LEADER + attr.Flags = quark.QQ_ALL_BACKENDS | quark.QQ_ENTRY_LEADER | quark.QQ_NO_SNAPSHOT qq, err := quark.OpenQueue(attr, 64) if err != nil { return nil, fmt.Errorf("open queue: %w", err) } var qqMtx sync.Mutex + var mtx sync.Mutex p := prvdr{ - ctx: ctx, - logger: logger, - qq: qq, - qqMtx: &qqMtx, - } - - go func(qq *quark.Queue, logger *logp.Logger, p *prvdr) { - for { + ctx: ctx, + logger: logger, + qq: qq, + qqMtx: &qqMtx, + combinedWait: 0 * time.Millisecond, + inBackoff: false, + backoffStart: time.Now(), + since: time.Now(), + backoffSkipped: 0, + mtx: &mtx, + } + + go func(ctx context.Context, qq *quark.Queue, logger *logp.Logger, p *prvdr) { + defer qq.Close() + for ctx.Err() == nil { p.qqMtx.Lock() events, err := qq.GetEvents() p.qqMtx.Unlock() if err != nil { - logger.Errorf("get events from quark: %w", err) + logger.Errorw("get events from quark, no more process enrichment from this processor will be done", "error", err) break } - for _, event := range events { - logger.Infof("event: %v", event) - } if len(events) == 0 { err = qq.Block() if err != nil { - logger.Errorf("quark block: %w", err) + logger.Errorw("quark block, no more process enrichment from this processor will be done", "error", err) break } } } - }(qq, logger, &p) + }(ctx, qq, logger, &p) bootID, _ = readBootID() pidNsInode, _ = readPIDNsInode() @@ -144,41 +152,36 @@ const ( resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset ) -var ( - combinedWait = 0 * time.Millisecond - inBackoff = false - backoffStart = time.Now() - since = time.Now() - backoffSkipped = 0 -) +func (p prvdr) SyncDB(_ *beat.Event, pid uint32) error { + p.mtx.Lock() + defer p.mtx.Unlock() -func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { if _, found := p.lookupLocked(pid); found { return nil } now := time.Now() - if inBackoff { - if now.Sub(backoffStart) > backoffDuration { - p.logger.Warnf("ended backoff, skipped %d processes", backoffSkipped) - inBackoff = false - combinedWait = 0 * time.Millisecond + if p.inBackoff { + if now.Sub(p.backoffStart) > backoffDuration { + p.logger.Warnw("ended backoff, skipped processes", "backoffSkipped", p.backoffSkipped) + p.inBackoff = false + p.combinedWait = 0 * time.Millisecond } else { - backoffSkipped += 1 + p.backoffSkipped += 1 return nil } } else { - if combinedWait > combinedWaitLimit { + if p.combinedWait > combinedWaitLimit { p.logger.Warn("starting backoff") - inBackoff = true - backoffStart = now - backoffSkipped = 0 + p.inBackoff = true + p.backoffStart = now + p.backoffSkipped = 0 return nil } // maintain a moving window of time for the delays we track - if now.Sub(since) > resetDuration { - since = now - combinedWait = 0 * time.Millisecond + if now.Sub(p.since) > resetDuration { + p.since = now + p.combinedWait = 0 * time.Millisecond } } @@ -187,15 +190,13 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { for { waited := time.Since(start) if _, found := p.lookupLocked(pid); found { - p.logger.Debugf("got process that was missing after %v", waited) - combinedWait = combinedWait + waited + p.logger.Debugw("got process that was missing ", "waited", waited) + p.combinedWait = p.combinedWait + waited return nil } if waited >= maxWaitLimit { - e := fmt.Errorf("process %v was not seen after %v", pid, waited) - p.logger.Warnf("%w", e) - combinedWait = combinedWait + waited - return e + p.combinedWait = p.combinedWait + waited + return fmt.Errorf("process %v was not seen after %v", pid, waited) } time.Sleep(nextWait) if nextWait*2+waited > maxWaitLimit { @@ -480,7 +481,7 @@ func basename(pathStr string) string { return "" } - return path.Base(pathStr) + return filepath.Base(pathStr) } // getUserName will return the name associated with the user ID, if it exists @@ -504,19 +505,19 @@ func getGroupName(id string) (string, bool) { func getEntryTypeName(entryType uint32) string { switch int(entryType) { case quark.QUARK_ELT_INIT: - return string(Init) + return Init case quark.QUARK_ELT_SSHD: - return string(Sshd) + return Sshd case quark.QUARK_ELT_SSM: - return string(Ssm) + return Ssm case quark.QUARK_ELT_CONTAINER: - return string(Container) + return Container case quark.QUARK_ELT_TERM: - return string(Terminal) + return Terminal case quark.QUARK_ELT_CONSOLE: - return string(EntryConsole) + return EntryConsole case quark.QUARK_ELT_KTHREAD: - return "kthread" + return Kthread default: return "unknown" } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_other.go b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_other.go similarity index 97% rename from x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_other.go rename to x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_other.go index 7e0f27bd53d..e895a696747 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/modernprovider/modernprovider_other.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_other.go @@ -4,7 +4,7 @@ //go:build linux && !((amd64 || arm64) && cgo) -package modernprovider +package kerneltracingprovider import ( "context" diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index 4d050ace2c5..5df70c120b5 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -54,17 +54,17 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { switch syscall { case "execveat", "execve": pe := types.ProcessExecEvent{} - proc_info, err := p.reader.GetProcess(pid) + procInfo, err := p.reader.GetProcess(pid) if err == nil { - pe.PIDs = proc_info.PIDs - pe.Creds = proc_info.Creds - pe.CTTY = proc_info.CTTY - pe.CWD = proc_info.Cwd - pe.Argv = proc_info.Argv - pe.Env = proc_info.Env - pe.Filename = proc_info.Filename + pe.PIDs = procInfo.PIDs + pe.Creds = procInfo.Creds + pe.CTTY = procInfo.CTTY + pe.CWD = procInfo.Cwd + pe.Argv = procInfo.Argv + pe.Env = procInfo.Env + pe.Filename = procInfo.Filename } else { - p.logger.Warnf("couldn't get process info from proc for pid %v: %w", pid, err) + p.logger.Warnw("couldn't get process info from proc for pid %v: %v", "pid", pid, "error", err) // If process info couldn't be taken from procfs, populate with as much info as // possible from the event pe.PIDs.Tgid = pid From b2eb7dd388232dd40890f9c8e43abdc42bc17de3 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 15 Oct 2024 18:24:06 -0700 Subject: [PATCH 43/44] change to pointer receiver --- .../kerneltracingprovider/kerneltracingprovider_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go index 3c2a9f9b79f..5ffcb570bee 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go @@ -152,7 +152,7 @@ const ( resetDuration = 5 * time.Second // After this amount of times with no backoffs, the combinedWait will be reset ) -func (p prvdr) SyncDB(_ *beat.Event, pid uint32) error { +func (p *prvdr) SyncDB(_ *beat.Event, pid uint32) error { p.mtx.Lock() defer p.mtx.Unlock() @@ -207,7 +207,7 @@ func (p prvdr) SyncDB(_ *beat.Event, pid uint32) error { } } -func (p prvdr) GetProcess(pid uint32) (*types.Process, error) { +func (p *prvdr) GetProcess(pid uint32) (*types.Process, error) { proc, found := p.lookupLocked(pid) if !found { return nil, fmt.Errorf("PID %d not found in cache", pid) From 874902cca8e40dda408d77f3fcd4a261eee65eba Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 15 Oct 2024 19:49:12 -0700 Subject: [PATCH 44/44] more review feedback changes --- .../sessionmd/add_session_metadata.go | 6 ++-- .../kerneltracingprovider_linux.go | 36 ++++++++++--------- .../provider/procfsprovider/procfsprovider.go | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 4e08c1b8fc0..28ef4697b79 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -13,6 +13,7 @@ import ( "strconv" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" @@ -46,6 +47,7 @@ type addSessionMetadata struct { } func New(cfg *cfg.C) (beat.Processor, error) { + cfgwarn.Beta("add_session_metadata processor is a beta feature.") c := defaultConfig() if err := cfg.Unpack(&c); err != nil { return nil, fmt.Errorf("fail to unpack the %v configuration: %w", processorName, err) @@ -166,7 +168,7 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { proc, err := p.provider.GetProcess(pid) if err != nil { e := fmt.Errorf("pid %v not found in db: %w", pid, err) - p.logger.Errorf("%v", e) + p.logger.Warnw("PID not found in provider", "pid", pid, "error", err) return nil, e } fullProcess = *proc @@ -174,7 +176,7 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { fullProcess, err = p.db.GetProcess(pid) if err != nil { e := fmt.Errorf("pid %v not found in db: %w", pid, err) - p.logger.Errorf("%v", e) + p.logger.Warnw("PID not found in provider", "pid", pid, "error", err) return nil, e } } diff --git a/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go index 5ffcb570bee..966f4b36c30 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/kerneltracingprovider/kerneltracingprovider_linux.go @@ -36,7 +36,6 @@ type prvdr struct { backoffStart time.Time since time.Time backoffSkipped int - mtx *sync.Mutex } type TTYType int @@ -86,11 +85,11 @@ func readPIDNsInode() (uint64, error) { pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") if err != nil { - return 0, fmt.Errorf("could not read /proc/self/ns/pid, process entity IDs will not be correct: %w", err) + return 0, fmt.Errorf("could not read /proc/self/ns/pid: %w", err) } if _, err = fmt.Sscanf(pidNsInodeRaw, "pid:[%d]", &ret); err != nil { - return 0, fmt.Errorf("could not parse contents of /proc/self/ns/pid (%s)process entity IDs will not be correct: %w", pidNsInodeRaw, err) + return 0, fmt.Errorf("could not parse contents of /proc/self/ns/pid (%q): %w", pidNsInodeRaw, err) } return ret, nil @@ -104,19 +103,16 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e return nil, fmt.Errorf("open queue: %w", err) } - var qqMtx sync.Mutex - var mtx sync.Mutex - p := prvdr{ + p := &prvdr{ ctx: ctx, logger: logger, qq: qq, - qqMtx: &qqMtx, + qqMtx: new(sync.Mutex), combinedWait: 0 * time.Millisecond, inBackoff: false, backoffStart: time.Now(), since: time.Now(), backoffSkipped: 0, - mtx: &mtx, } go func(ctx context.Context, qq *quark.Queue, logger *logp.Logger, p *prvdr) { @@ -137,12 +133,18 @@ func NewProvider(ctx context.Context, logger *logp.Logger) (provider.Provider, e } } } - }(ctx, qq, logger, &p) + }(ctx, qq, logger, p) - bootID, _ = readBootID() - pidNsInode, _ = readPIDNsInode() + bootID, err = readBootID() + if err != nil { + p.logger.Errorw("failed to read boot ID, entity ID will not be correct", "error", err) + } + pidNsInode, err = readPIDNsInode() + if err != nil { + p.logger.Errorw("failed to read PID namespace inode, entity ID will not be correct", "error", err) + } - return &p, nil + return p, nil } const ( @@ -153,10 +155,12 @@ const ( ) func (p *prvdr) SyncDB(_ *beat.Event, pid uint32) error { - p.mtx.Lock() - defer p.mtx.Unlock() + p.qqMtx.Lock() + defer p.qqMtx.Unlock() + + // Use qq.Lookup, not lookupLocked, in this function. Mutex is locked for entire function - if _, found := p.lookupLocked(pid); found { + if _, found := p.qq.Lookup(int(pid)); found { return nil } @@ -189,7 +193,7 @@ func (p *prvdr) SyncDB(_ *beat.Event, pid uint32) error { nextWait := 5 * time.Millisecond for { waited := time.Since(start) - if _, found := p.lookupLocked(pid); found { + if _, found := p.qq.Lookup(int(pid)); found { p.logger.Debugw("got process that was missing ", "waited", waited) p.combinedWait = p.combinedWait + waited return nil diff --git a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go index 5df70c120b5..4934a79fc52 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/procfsprovider/procfsprovider.go @@ -64,7 +64,7 @@ func (p prvdr) SyncDB(ev *beat.Event, pid uint32) error { pe.Env = procInfo.Env pe.Filename = procInfo.Filename } else { - p.logger.Warnw("couldn't get process info from proc for pid %v: %v", "pid", pid, "error", err) + p.logger.Warnw("couldn't get process info from proc for pid", "pid", pid, "error", err) // If process info couldn't be taken from procfs, populate with as much info as // possible from the event pe.PIDs.Tgid = pid