Skip to content

Commit

Permalink
API: Add Socket ID, ProcessPath; Xapture existing TLS connections (#120)
Browse files Browse the repository at this point in the history
  • Loading branch information
iluxa authored Dec 5, 2024
1 parent 63c24ca commit 03ec8d4
Show file tree
Hide file tree
Showing 12 changed files with 187 additions and 32 deletions.
28 changes: 28 additions & 0 deletions bpf/events.c
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,16 @@ void BPF_KPROBE(tcp_connect) {
return;
}

u64 inode = 0;
struct socket *s = BPF_CORE_READ(sk, sk_socket);
if (s) {
struct file *sock_file = BPF_CORE_READ(s, file);
if (sock_file) {
inode = BPF_CORE_READ(sock_file, f_inode, i_ino);
}
}


struct task_struct* task = (struct task_struct*)bpf_get_current_task();
struct syscall_event ev = {
.event_id = SYSCALL_EVENT_ID_CONNECT,
Expand All @@ -40,6 +50,7 @@ void BPF_KPROBE(tcp_connect) {
.parent_pid = get_task_pid(get_parent_task(task)),
.host_pid = BPF_CORE_READ(task, tgid),
.host_parent_pid = get_parent_task_pid(task),
.inode_id = inode,
};

if (read_addrs_ports(ctx, (struct sock*)PT_REGS_PARM1(ctx), &ev.ip_src, &ev.port_src, &ev.ip_dst, &ev.port_dst)) {
Expand Down Expand Up @@ -84,6 +95,12 @@ void BPF_KRETPROBE(syscall__accept4_ret) {
return;
}

u64 inode = 0;
struct file *sock_file = BPF_CORE_READ(sock, file);
if (sock_file) {
inode = BPF_CORE_READ(sock_file, f_inode, i_ino);
}

struct task_struct* task = (struct task_struct*)bpf_get_current_task();

struct syscall_event ev = {
Expand All @@ -93,6 +110,7 @@ void BPF_KRETPROBE(syscall__accept4_ret) {
.parent_pid = get_task_pid(get_parent_task(task)),
.host_pid = BPF_CORE_READ(task, tgid),
.host_parent_pid = get_parent_task_pid(task),
.inode_id = inode,
};

if (read_addrs_ports(ctx, sk, &ev.ip_dst, &ev.port_dst, &ev.ip_src, &ev.port_src)) {
Expand Down Expand Up @@ -163,6 +181,15 @@ void BPF_KPROBE(tcp_close) {
return;
}

u64 inode = 0;
struct socket *s = BPF_CORE_READ(sk, sk_socket);
if (s) {
struct file *sock_file = BPF_CORE_READ(s, file);
if (sock_file) {
inode = BPF_CORE_READ(sock_file, f_inode, i_ino);
}
}

__u16 event = 0;

__u64 key = (__u64)sk;
Expand All @@ -184,6 +211,7 @@ void BPF_KPROBE(tcp_close) {
.parent_pid = get_task_pid(get_parent_task(task)),
.host_pid = BPF_CORE_READ(task, tgid),
.host_parent_pid = get_parent_task_pid(task),
.inode_id = inode,
};

if (read_addrs_ports(ctx, (struct sock*)PT_REGS_PARM1(ctx), &ev.ip_src, &ev.port_src, &ev.ip_dst, &ev.port_dst)) {
Expand Down
1 change: 1 addition & 0 deletions bpf/include/events.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ struct syscall_event {
char comm[16];

__u64 cgroup_id;
__u64 inode_id;

__be32 ip_src;
__be32 ip_dst;
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
github.com/hashicorp/golang-lru v0.5.4
github.com/hashicorp/golang-lru/v2 v2.0.2
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9
github.com/kubeshark/api v1.1.31
github.com/kubeshark/api v1.1.32
github.com/kubeshark/gopacket v1.1.30
github.com/kubeshark/tracerproto v1.0.3-0.20240730073449-de3a99a3719c
github.com/kubeshark/utils v0.0.0-20240912210808-416dca610f11
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubeshark/api v1.1.31 h1:stOsatiJ8C3SKLIStAwykc/dybHlyy9fYyH8b9tIv1c=
github.com/kubeshark/api v1.1.31/go.mod h1:+Ua35OiwreWiUYfqJz0Aswn3UmsLctLUQh3tvxagQz4=
github.com/kubeshark/api v1.1.32 h1:qZ4so2FBJOgw7Eqmlvr1Kz29hALM4hsFgHzC/cULoMM=
github.com/kubeshark/api v1.1.32/go.mod h1:+Ua35OiwreWiUYfqJz0Aswn3UmsLctLUQh3tvxagQz4=
github.com/kubeshark/gopacket v1.1.30 h1:Dz6eo7b6+NdVCrgiyKxlGEVTm0L6PwgbVvSomsuwIyU=
github.com/kubeshark/gopacket v1.1.30/go.mod h1:Qo8/i/tdT74CCT7/pjO0L55Pktv5dQfj7M/Arv8MKm8=
github.com/kubeshark/tracerproto v1.0.0/go.mod h1:+efDYkwXxwakmHRpxHVEekyXNtg/aFx0uSo/I0lGV9k=
Expand Down
22 changes: 19 additions & 3 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import (

"github.com/kubeshark/tracer/misc"
"github.com/kubeshark/tracer/pkg/kubernetes"
"github.com/kubeshark/tracer/pkg/resolver"
"github.com/kubeshark/tracer/pkg/utils"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"k8s.io/apimachinery/pkg/types"
Expand Down Expand Up @@ -112,13 +114,26 @@ func run() {
select {}
}

isCgroupsV2, err := utils.IsCgroupV2()
if err != nil {
log.Error().Err(err).Msg("detect cgroup version failed")
return
}

tcpMap, err := resolver.GatherPidsTCPMap(*procfs, isCgroupsV2)
if err != nil {
log.Error().Err(err).Msg("tcp map lookup failed")
return
}

tracer = &Tracer{
procfs: *procfs,
targetedCgroupIDs: map[uint64]struct{}{},
runningPods: make(map[types.UID]podInfo),
tcpMap: tcpMap,
}

_, err := rest.InClusterConfig()
_, err = rest.InClusterConfig()
clusterMode := err == nil
errOut := make(chan error, 100)
go func() {
Expand All @@ -143,7 +158,7 @@ func run() {
}
misc.InitDataDir()

err = createTracer()
err = createTracer(isCgroupsV2)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't initialize the tracer:")
}
Expand Down Expand Up @@ -173,14 +188,15 @@ func stop() {
}
}

func createTracer() (err error) {
func createTracer(isCgroupsV2 bool) (err error) {
chunksBufferSize := os.Getpagesize() * 10000
logBufferSize := os.Getpagesize()

if err = tracer.Init(
chunksBufferSize,
logBufferSize,
*procfs,
isCgroupsV2,
); err != nil {
return
}
Expand Down
11 changes: 7 additions & 4 deletions pkg/discoverer/pids.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,19 +96,19 @@ func (p *pids) targetCgroup(cgroupId uint64) {
if err != nil {
// process can be already terminated
log.Debug().Err(err).Uint32("pid", pid).Uint64("cgroup", pi.cgroupId).Msg("Open executable failed")
return
continue
}

offsets, err := goHooks.FindGoOffsets(pi.goPath)
if err != nil {
return
continue
}
hook := goHooks.GoHooks{}

err = hook.InstallHooks(p.bpfObjs, ex, offsets)
if err != nil {
log.Debug().Uint32("pid", pid).Uint64("cgroup", cgroupId).Msg(fmt.Sprintf("install go hook failed: %v", err))
return
continue
}
pi.goHook = &hook

Expand All @@ -121,7 +121,7 @@ func (p *pids) targetCgroup(cgroupId uint64) {
err := hook.InstallUprobes(p.bpfObjs, pi.sslPath)
if err != nil {
log.Debug().Err(err).Uint32("pid", pid).Uint64("cgroup", cgroupId).Msg("install ssl hook failed")
return
continue
}
pi.sslHook = &hook

Expand Down Expand Up @@ -232,6 +232,9 @@ func (p *pids) installGoHook(e foundPidEvent) (*goHooks.GoHooks, string) {

if filepath.Base(path) == "envoy" {
p.envoyPath = path
defer func() {
p.envoyPath = ""
}()
}

ex, err := link.OpenExecutable(path)
Expand Down
2 changes: 2 additions & 0 deletions pkg/events/syscalls.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ type SyscallEventMessage struct {
Command [16]byte

CgroupID uint64
SocketID uint64

IpSrc uint32
IpDst uint32
Expand All @@ -22,4 +23,5 @@ type SyscallEventMessage struct {
type SyscallEvent struct {
SyscallEventMessage
ContainerID string
ProcessPath string
}
3 changes: 3 additions & 0 deletions pkg/poller/syscall/tracer_syscall_events.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@ import (
"fmt"
"net"
"os"
"path/filepath"

"github.com/cilium/ebpf/perf"
"github.com/kubeshark/tracer/misc"
"github.com/kubeshark/tracer/pkg/bpf"
"github.com/kubeshark/tracer/pkg/cgroup"
"github.com/kubeshark/tracer/pkg/events"
"github.com/kubeshark/tracer/pkg/resolver"
"github.com/kubeshark/tracer/socket"
"github.com/rs/zerolog/log"
)
Expand Down Expand Up @@ -99,6 +101,7 @@ func (t *SyscallEventsTracer) pollEvents() {
var e events.SyscallEvent
e.SyscallEventMessage = ev

e.ProcessPath, _ = resolver.ResolveSymlinkWithoutValidation(filepath.Join("/hostproc", fmt.Sprintf("%v", ev.HostPid), "exe"))
log.Debug().Msg(fmt.Sprintf("Syscall event %v: %v:%v->%v:%v command: %v host pid: %v host ppid: %v pid: %v ppid: %v cgroup id: %v",
evName,
toIP(e.IpSrc),
Expand Down
91 changes: 91 additions & 0 deletions pkg/resolver/connections.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package resolver

import (
"encoding/binary"
"fmt"

"github.com/rs/zerolog/log"
)

type ip4Port struct {
ip uint32
port uint16
}

type ConnectionCb func(pidFd uint64, isClient bool) error

func GatherPidsTCPMap(procfs string, isCgroupV2 bool) (tcpMap map[uint64]bool, err error) {
tcpMap = make(map[uint64]bool) // (pid<<32)|fd -> client flag

cgroups, err := getAllCgroups(procfs, isCgroupV2)
if err != nil {
log.Error().Err(err).Msg("get all cgroups failed")
return
}
log.Info().Int("cgroups", len(cgroups)).Msg("got cgroups")
for cgroup := range cgroups {
pids, err := findPIDsInCgroup(procfs, isCgroupV2, cgroup)
if err != nil {
log.Debug().Msg(fmt.Sprintf("get pids for cgroup %q failed: %v", cgroup, err))
continue
}

log.Debug().Int("connections", len(pids)).Str("Cgroup", cgroup).Msg("got pids")

var conns []IpSocketLine
for i := range pids {
conns, err = getTCPConnections(fmt.Sprintf("%v", pids[i].hostPid))
if err != nil {
// process can be short-living
continue
}
break
}

log.Debug().Int("connections", len(conns)).Str("Cgroup", cgroup).Msg("got connections")

// ip:port -> inode:
listenConnections := make(map[uint16]uint32)

// (pid<<32) | fd -> local ip:port
// for established connections
pidConnections := make(map[uint64]ip4Port)

for _, conn := range conns {
if conn.Inode == 0 {
continue
}
if conn.St == 0xA { // listen sockets
ipPort := ip4Port{
ip: binary.BigEndian.Uint32(conn.LocalAddr.To4()),
port: uint16(conn.LocalPort),
}
listenConnections[ipPort.port] = ipPort.ip
} else if conn.St == 0x1 { // established sockets
for _, pid := range pids {
if fd, ok := pid.socketInodes[conn.Inode]; ok {
pidFd := uint64(pid.hostPid)
pidFd = (pidFd << 32) | uint64(fd)
ipPort := ip4Port{
ip: binary.BigEndian.Uint32(conn.LocalAddr.To4()),
port: uint16(conn.LocalPort),
}
pidConnections[pidFd] = ipPort
}
}
}

for pidFd, ipPort := range pidConnections {
isClient := true
if listenIp, ok := listenConnections[ipPort.port]; ok {
// socket can be bound on 0.0.0.0 or interface address
if listenIp == 0 || listenIp == ipPort.ip {
isClient = false
}
}
tcpMap[pidFd] = isClient
}
}
}
return
}
11 changes: 9 additions & 2 deletions pkg/resolver/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ type pidInfo struct {
hostParentPid uint32
name string
path string
socketInodes map[uint64]struct{}
socketInodes map[uint64]uint32 // inode -> fd
}

type connectionsMap map[string]*connectionResolution
Expand All @@ -101,6 +101,7 @@ func resolvePair(connMap connectionsMap, localIP, localPort, remoteIP, remotePor
HostProcessID: int(res.HostProcessID),
HostParentProcessID: int(res.HostParentProcessID),
ProcessName: res.ProcessName,
ProcessPath: res.ProcessPath,
}
log.Debug().Str("local IP", localIP).Str("local Port", localPort).Str("remote IP", remoteIP).Str("remote Port", remotePort).Interface("resolution", r).Msg("found resolution")
return &r
Expand Down Expand Up @@ -168,6 +169,12 @@ func getAllFlows(procfs string, isCgroupV2 bool, proto string) connectionsMap {
log.Debug().Int("connections", len(conns)).Str("Cgroup", cgroup).Str("proto", proto).Msg("got connections")

for _, conn := range conns {

// pass only established connections
if conn.St != 1 {
continue
}

if conn.Inode == 0 {
continue
}
Expand Down Expand Up @@ -331,7 +338,7 @@ func getPidInfo(proc string, hostPid uint32, isCgroupV2 bool) (pi pidInfo, err e
return
}

if pi.path, err = resolveSymlinkWithoutValidation(filepath.Join(proc, fmt.Sprintf("%v", hostPid), "exe")); err != nil {
if pi.path, err = ResolveSymlinkWithoutValidation(filepath.Join(proc, fmt.Sprintf("%v", hostPid), "exe")); err != nil {
return
}

Expand Down
Loading

0 comments on commit 03ec8d4

Please sign in to comment.