From d791d02fac5c318295bd0a4116c3fa81aef3a638 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Thu, 2 Jan 2025 08:38:37 +0000 Subject: [PATCH 01/13] feat: Introduce github.com/containerd/containerd/pkg/cap to check whether process has CAP_BPF privilege Signed-off-by: spencercjh --- agent/agent.go | 4 ++-- agent/common/permission.go | 15 +++++++++++++++ go.mod | 2 ++ go.sum | 4 ++++ 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 agent/common/permission.go diff --git a/agent/agent.go b/agent/agent.go index 6652e3f3..6181f32b 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -35,8 +35,8 @@ func SetupAgent(options ac.AgentOptions) { return } - if os.Geteuid() != 0 { - common.AgentLog.Error("Kyanos requires root privileges to run. Please run kyanos with sudo.") + if ok, err := ac.HasPermission(); err != nil || !ok { + common.AgentLog.Error("Kyanos requires CAP_BPF to run. Please run kyanos with sudo or give correct capability.") return } diff --git a/agent/common/permission.go b/agent/common/permission.go new file mode 100644 index 00000000..499cca3b --- /dev/null +++ b/agent/common/permission.go @@ -0,0 +1,15 @@ +package common + +import ( + "slices" + + "github.com/containerd/containerd/pkg/cap" +) + +func HasPermission() (bool, error) { + current, err := cap.Current() + if err != nil { + return false, err + } + return slices.Contains(current, "CAP_BPF"), nil +} diff --git a/go.mod b/go.mod index c852d663..875fcbc3 100644 --- a/go.mod +++ b/go.mod @@ -139,6 +139,8 @@ require ( k8s.io/apiserver v0.31.1 // indirect k8s.io/component-base v0.31.1 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + kernel.org/pub/linux/libs/security/libcap/cap v1.2.73 // indirect + kernel.org/pub/linux/libs/security/libcap/psx v1.2.73 // indirect rsc.io/binaryregexp v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index d691883e..5990a8a9 100644 --- a/go.sum +++ b/go.sum @@ -435,6 +435,10 @@ k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20230824000246-2cb31c9333ad k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20230824000246-2cb31c9333ad/go.mod h1:92B+ezOsw1+IWrnyKuKTKb9ioXD7xacrEml1MI6lv2I= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.73 h1:Th2b8jljYqkyZKS3aD3N9VpYsQpHuXLgea+SZUIfODA= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.73/go.mod h1:hbeKwKcboEsxARYmcy/AdPVN11wmT/Wnpgv4k4ftyqY= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.73 h1:SEAEUiPVylTD4vqqi+vtGkSnXeP2FcRO3FoZB1MklMw= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.73/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From 3fd7f57784d4c31ea944163bedd10c91f5da06be Mon Sep 17 00:00:00 2001 From: spencercjh Date: Thu, 2 Jan 2025 08:42:27 +0000 Subject: [PATCH 02/13] fix: better logs --- agent/agent.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 6181f32b..0b6e64e7 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -35,8 +35,11 @@ func SetupAgent(options ac.AgentOptions) { return } - if ok, err := ac.HasPermission(); err != nil || !ok { - common.AgentLog.Error("Kyanos requires CAP_BPF to run. Please run kyanos with sudo or give correct capability.") + if ok, err := ac.HasPermission(); err != nil { + common.AgentLog.Error("check capabilities failed: ", err) + return + } else if !ok { + common.AgentLog.Error("Kyanos requires CAP_BPF to run. Please run kyanos with sudo or run container in privilege mode.") return } From 10b7bdbdedd0a450a51d008f892cca64a154f0f1 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Thu, 2 Jan 2025 09:05:43 +0000 Subject: [PATCH 03/13] fix: adapt to e2e test env --- agent/common/permission.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/agent/common/permission.go b/agent/common/permission.go index 499cca3b..7776cc14 100644 --- a/agent/common/permission.go +++ b/agent/common/permission.go @@ -1,12 +1,17 @@ package common import ( + "os" "slices" "github.com/containerd/containerd/pkg/cap" ) func HasPermission() (bool, error) { + // root is considered as having CAP_BPF capability. + if os.Geteuid() == 0 { + return true, nil + } current, err := cap.Current() if err != nil { return false, err From d1bf62a2c7b99aff897cc33c67527e9d46bd4499 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Thu, 2 Jan 2025 09:14:58 +0000 Subject: [PATCH 04/13] style: go mod tidy --- agent/common/permission.go | 5 ----- go.mod | 4 +--- go.sum | 6 ------ 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/agent/common/permission.go b/agent/common/permission.go index 7776cc14..499cca3b 100644 --- a/agent/common/permission.go +++ b/agent/common/permission.go @@ -1,17 +1,12 @@ package common import ( - "os" "slices" "github.com/containerd/containerd/pkg/cap" ) func HasPermission() (bool, error) { - // root is considered as having CAP_BPF capability. - if os.Geteuid() == 0 { - return true, nil - } current, err := cap.Current() if err != nil { return false, err diff --git a/go.mod b/go.mod index 875fcbc3..a913d8e6 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( k8s.io/cri-api v0.31.0 k8s.io/klog/v2 v2.130.1 k8s.io/kubernetes v1.24.17 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 ) require ( @@ -138,9 +139,6 @@ require ( k8s.io/apimachinery v0.31.1 // indirect k8s.io/apiserver v0.31.1 // indirect k8s.io/component-base v0.31.1 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - kernel.org/pub/linux/libs/security/libcap/cap v1.2.73 // indirect - kernel.org/pub/linux/libs/security/libcap/psx v1.2.73 // indirect rsc.io/binaryregexp v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index 5990a8a9..8e9ecdb1 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= -github.com/charmbracelet/bubbletea v1.2.2 h1:EMz//Ky/aFS2uLcKqpCst5UOE6z5CFDGRsUpyXz0chs= -github.com/charmbracelet/bubbletea v1.2.2/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM= github.com/charmbracelet/bubbletea v1.2.4 h1:KN8aCViA0eps9SCOThb2/XPIlea3ANJLUkv3KnQRNCE= github.com/charmbracelet/bubbletea v1.2.4/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM= github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= @@ -435,10 +433,6 @@ k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20230824000246-2cb31c9333ad k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20230824000246-2cb31c9333ad/go.mod h1:92B+ezOsw1+IWrnyKuKTKb9ioXD7xacrEml1MI6lv2I= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kernel.org/pub/linux/libs/security/libcap/cap v1.2.73 h1:Th2b8jljYqkyZKS3aD3N9VpYsQpHuXLgea+SZUIfODA= -kernel.org/pub/linux/libs/security/libcap/cap v1.2.73/go.mod h1:hbeKwKcboEsxARYmcy/AdPVN11wmT/Wnpgv4k4ftyqY= -kernel.org/pub/linux/libs/security/libcap/psx v1.2.73 h1:SEAEUiPVylTD4vqqi+vtGkSnXeP2FcRO3FoZB1MklMw= -kernel.org/pub/linux/libs/security/libcap/psx v1.2.73/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From cd1133ebe27c4650013f0b7f2e593b67637de987 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Thu, 2 Jan 2025 11:14:06 +0000 Subject: [PATCH 05/13] fix: make tests pass --- .github/workflows/test.yml | 56 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c9dcc6c..b6e5755b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -165,9 +165,9 @@ jobs: cat /etc/issue pushd /host if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_filter_by_comm.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_filter_by_comm.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_filter_by_comm.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_filter_by_comm.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi popd @@ -181,9 +181,9 @@ jobs: cat /etc/issue pushd /host if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_gotls.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_gotls.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_gotls.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_gotls.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi popd @@ -197,9 +197,9 @@ jobs: cat /etc/issue pushd /host if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_https.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_https.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_https.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_https.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi popd @@ -213,9 +213,9 @@ jobs: cat /etc/issue pushd /host if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_side.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_side.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_side.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_side.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi popd @@ -229,9 +229,9 @@ jobs: cat /etc/issue pushd /host if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_mysql.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_mysql.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_mysql.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_mysql.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi popd @@ -244,9 +244,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_base.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_base.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_base.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_base.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi @@ -259,9 +259,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_filter_by_l4.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_filter_by_l4.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_filter_by_l4.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_filter_by_l4.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test kern evt @@ -273,9 +273,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_kern_evt.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_kern_evt.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_kern_evt.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_kern_evt.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test test docker filter by container id @@ -287,9 +287,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_docker_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_docker_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_docker_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_docker_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test test docker filter by container name @@ -301,9 +301,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_docker_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_docker_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_docker_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_docker_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test filter by pid @@ -315,9 +315,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_docker_filter_by_pid.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_docker_filter_by_pid.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_docker_filter_by_pid.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_docker_filter_by_pid.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test test containerd filter by container name @@ -329,9 +329,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_containerd_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_containerd_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_containerd_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_containerd_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test test containerd filter by container id @@ -343,9 +343,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_containerd_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_containerd_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_containerd_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_containerd_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test redis @@ -357,9 +357,9 @@ jobs: uname -a cat /etc/issue if [ -f "/var/lib/kyanos/btf/current.btf" ]; then - bash /host/testdata/test_redis.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' + bash /host/testdata/test_redis.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf' else - bash /host/testdata/test_redis.sh '/host/kyanos/kyanos $kyanos_log_option' + bash /host/testdata/test_redis.sh 'sudo /host/kyanos/kyanos $kyanos_log_option' fi - name: Test k8s From 4423b8c446f647e39ccf79e121d98e77326b7e71 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Thu, 2 Jan 2025 12:24:20 +0000 Subject: [PATCH 06/13] fix: DO NOT use containerd cap package --- agent/common/permission.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/agent/common/permission.go b/agent/common/permission.go index 499cca3b..3b5ab154 100644 --- a/agent/common/permission.go +++ b/agent/common/permission.go @@ -1,15 +1,14 @@ package common import ( - "slices" - - "github.com/containerd/containerd/pkg/cap" + "golang.org/x/sys/unix" ) func HasPermission() (bool, error) { - current, err := cap.Current() - if err != nil { + hdr := unix.CapUserHeader{Version: unix.LINUX_CAPABILITY_VERSION_3} + var data [2]unix.CapUserData + if err := unix.Capget(&hdr, &data[0]); err != nil { return false, err } - return slices.Contains(current, "CAP_BPF"), nil + return data[0].Permitted&unix.CAP_BPF != 0, nil } From 2674d64588f6cb24729bbaae29500a90ab62f788 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 3 Jan 2025 03:26:51 +0000 Subject: [PATCH 07/13] test: introduce tests to verify agent/common/permission.go --- .github/workflows/test.yml | 12 ++++++++++++ testdata/run_cap_bpf_test.sh | 24 ++++++++++++++++++++++++ testdata/test_add_cap_bpf.sh | 12 ++++++++++++ testdata/test_not_add_cap_bpf.sh | 12 ++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 testdata/run_cap_bpf_test.sh create mode 100644 testdata/test_add_cap_bpf.sh create mode 100644 testdata/test_not_add_cap_bpf.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6e5755b..8455d8a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,6 +155,18 @@ jobs: #install python pip sudo apt install -y python3 python3-pip pipx + - name: Test CAP_BPF privilege check + uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 + with: + provision: 'false' + cmd: | + set -euxo pipefail + uname -a + cat /etc/issue + pushd /host + bash /host/testdata/run_cap_bpf_test.sh "" + popd + - name: Test filter by comm uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 with: diff --git a/testdata/run_cap_bpf_test.sh b/testdata/run_cap_bpf_test.sh new file mode 100644 index 00000000..5568693c --- /dev/null +++ b/testdata/run_cap_bpf_test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -ex + +DOCKER_REGISTRY="$1" +if [ -n "$DOCKER_REGISTRY" ]; then + # 检查是否以 / 结尾 + if [[ "$DOCKER_REGISTRY" != */ ]]; then + DOCKER_REGISTRY="${DOCKER_REGISTRY}/" + fi +else + echo "DOCKER_REGISTRY is missing." +fi + +sudo docker run -d --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker cp /host/kyanos/kyanos alpine:/ +sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ +sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos"' +sudo docker stop alpine && sudo docker rm alpine + +sudo docker run -d --add-cap CAP_BPF --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker cp /host/kyanos/kyanos alpine:/ +sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ +sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos"' +sudo docker stop alpine && sudo docker rm alpine diff --git a/testdata/test_add_cap_bpf.sh b/testdata/test_add_cap_bpf.sh new file mode 100644 index 00000000..c295c9aa --- /dev/null +++ b/testdata/test_add_cap_bpf.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env sh +set -ex + +CMD="$1" +FILE_PREFIX="/tmp/kyanos" +LNAME="${FILE_PREFIX}_test_add_cap_bpf_before.log" + +timeout 30 ${CMD} watch http --debug-output 2>&1 | tee "${LNAME}" & +wait + +cat "${LNAME}" +cat "${LNAME}" | grep -v "requires CAP_BPF" \ No newline at end of file diff --git a/testdata/test_not_add_cap_bpf.sh b/testdata/test_not_add_cap_bpf.sh new file mode 100644 index 00000000..e38a19f3 --- /dev/null +++ b/testdata/test_not_add_cap_bpf.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env sh +set -ex + +CMD="$1" +FILE_PREFIX="/tmp/kyanos" +LNAME="${FILE_PREFIX}_test_not_add_cap_bpf_before.log" + +timeout 30 ${CMD} watch http --debug-output 2>&1 | tee "${LNAME}" & +wait + +cat "${LNAME}" +cat "${LNAME}" | grep "requires CAP_BPF" From cacbb126a7377e16a7bd360ae8425f9cd7fca186 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 3 Jan 2025 08:26:28 +0000 Subject: [PATCH 08/13] fix: correct implementation refer to https://man7.org/linux/man-pages/man2/capset.2.html --- agent/common/permission.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/agent/common/permission.go b/agent/common/permission.go index 3b5ab154..a09fc64b 100644 --- a/agent/common/permission.go +++ b/agent/common/permission.go @@ -4,11 +4,21 @@ import ( "golang.org/x/sys/unix" ) +const ( + // capBpf 0000 0000 0000 0000 0000 0000 1000 0000 + capBpf = 1 << (unix.CAP_BPF - 32) + // capSysAdmin 0000 0000 0010 0000 0000 0000 0000 0000 + capSysAdmin = 1 << unix.CAP_SYS_ADMIN +) + +// HasPermission reference: https://man7.org/linux/man-pages/man2/capset.2.html func HasPermission() (bool, error) { hdr := unix.CapUserHeader{Version: unix.LINUX_CAPABILITY_VERSION_3} var data [2]unix.CapUserData if err := unix.Capget(&hdr, &data[0]); err != nil { return false, err } - return data[0].Permitted&unix.CAP_BPF != 0, nil + // Note that the CAP_* values are bit indexes and need to be bit-shifted before ORing into the bit fields. + // Note that 64-bit capabilities use datap[0] and datap[1], whereas 32-bit capabilities use only datap[0]. + return data[1].Permitted&capBpf != 0 || data[0].Permitted&capSysAdmin != 0, nil } From ebc5112f91250d02e81d0272a4e9841458e9f3bd Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 3 Jan 2025 08:27:53 +0000 Subject: [PATCH 09/13] test: test test_add_cap_bpf first --- testdata/run_cap_bpf_test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testdata/run_cap_bpf_test.sh b/testdata/run_cap_bpf_test.sh index 5568693c..eb9bc750 100644 --- a/testdata/run_cap_bpf_test.sh +++ b/testdata/run_cap_bpf_test.sh @@ -11,14 +11,14 @@ else echo "DOCKER_REGISTRY is missing." fi -sudo docker run -d --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker run -d --cap-add CAP_BPF --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ -sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ -sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos"' +sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ +sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos"' sudo docker stop alpine && sudo docker rm alpine -sudo docker run -d --add-cap CAP_BPF --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker run -d --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ -sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ -sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos"' +sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ +sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos"' sudo docker stop alpine && sudo docker rm alpine From d9ac045498e501a9276789623fae3182794d1196 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 3 Jan 2025 08:50:56 +0000 Subject: [PATCH 10/13] test: cap-add difference capability for different kernal --- .github/workflows/test.yml | 17 ++++++++++++++++- testdata/run_cap_bpf_test.sh | 5 +++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8455d8a5..75fee55c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -157,6 +157,7 @@ jobs: - name: Test CAP_BPF privilege check uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 + if: ${{ !contains(fromJSON('["4.19-20240912.022020", "5.4-20240912.022020"]'), matrix.kernel) }} with: provision: 'false' cmd: | @@ -164,7 +165,21 @@ jobs: uname -a cat /etc/issue pushd /host - bash /host/testdata/run_cap_bpf_test.sh "" + bash /host/testdata/run_cap_bpf_test.sh "" "CAP_BPF" + popd + + + - name: Test CAP_SYS_ADMIN privilege check + uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19 + if: contains(fromJSON('["4.19-20240912.022020", "5.4-20240912.022020"]'), matrix.kernel) + with: + provision: 'false' + cmd: | + set -euxo pipefail + uname -a + cat /etc/issue + pushd /host + bash /host/testdata/run_cap_bpf_test.sh "" "CAP_SYS_ADMIN" popd - name: Test filter by comm diff --git a/testdata/run_cap_bpf_test.sh b/testdata/run_cap_bpf_test.sh index eb9bc750..cc79e37d 100644 --- a/testdata/run_cap_bpf_test.sh +++ b/testdata/run_cap_bpf_test.sh @@ -10,14 +10,15 @@ if [ -n "$DOCKER_REGISTRY" ]; then else echo "DOCKER_REGISTRY is missing." fi +CAP_TO_ADD="$2" -sudo docker run -d --cap-add CAP_BPF --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=$CAP_TO_ADD --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos"' sudo docker stop alpine && sudo docker rm alpine -sudo docker run -d --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker run -d --ulimit memlock=100000000000:100000000000 --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos"' From 6d9ca3486b3b0bc4a01d311ddb42a546393fca4e Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 3 Jan 2025 09:14:20 +0000 Subject: [PATCH 11/13] test: load btf file to container and run kyanos with --btf flag --- testdata/run_cap_bpf_test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testdata/run_cap_bpf_test.sh b/testdata/run_cap_bpf_test.sh index cc79e37d..38f6743f 100644 --- a/testdata/run_cap_bpf_test.sh +++ b/testdata/run_cap_bpf_test.sh @@ -15,11 +15,13 @@ CAP_TO_ADD="$2" sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=$CAP_TO_ADD --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ -sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos"' +sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/ +sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos --btf /current.btf"' sudo docker stop alpine && sudo docker rm alpine sudo docker run -d --ulimit memlock=100000000000:100000000000 --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ -sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos"' +sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/ +sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos --btf /current.btf"' sudo docker stop alpine && sudo docker rm alpine From c480f195d14480a945d0a53a36a694470479cc9c Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 3 Jan 2025 09:23:30 +0000 Subject: [PATCH 12/13] test: add missing capability CAP_SYS_RESOURCE --- testdata/run_cap_bpf_test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testdata/run_cap_bpf_test.sh b/testdata/run_cap_bpf_test.sh index 38f6743f..7235305d 100644 --- a/testdata/run_cap_bpf_test.sh +++ b/testdata/run_cap_bpf_test.sh @@ -12,16 +12,16 @@ else fi CAP_TO_ADD="$2" -sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=$CAP_TO_ADD --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=CAP_SYS_RESOURCE --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ -sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ +sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/ -sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos --btf /current.btf"' +sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos --btf /current.btf"' sudo docker stop alpine && sudo docker rm alpine -sudo docker run -d --ulimit memlock=100000000000:100000000000 --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=$CAP_TO_ADD --cap-add=CAP_SYS_RESOURCE --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ -sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ +sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/ -sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos --btf /current.btf"' +sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos --btf /current.btf"' sudo docker stop alpine && sudo docker rm alpine From 97b3013849fe7ed1d4e58a46b4b66c03c8b939f9 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Fri, 3 Jan 2025 09:45:16 +0000 Subject: [PATCH 13/13] test: try to use --privileged instead of cap-add --- testdata/run_cap_bpf_test.sh | 5 +++-- testdata/test_add_cap_bpf.sh | 2 +- testdata/test_not_add_cap_bpf.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/testdata/run_cap_bpf_test.sh b/testdata/run_cap_bpf_test.sh index 7235305d..fc2d8262 100644 --- a/testdata/run_cap_bpf_test.sh +++ b/testdata/run_cap_bpf_test.sh @@ -10,8 +10,8 @@ if [ -n "$DOCKER_REGISTRY" ]; then else echo "DOCKER_REGISTRY is missing." fi -CAP_TO_ADD="$2" +# CAP_SYS_RESOURCE reference: https://docs.ebpf.io/linux/concepts/resource-limit/ sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=CAP_SYS_RESOURCE --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/ @@ -19,7 +19,8 @@ sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/ sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos --btf /current.btf"' sudo docker stop alpine && sudo docker rm alpine -sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=$CAP_TO_ADD --cap-add=CAP_SYS_RESOURCE --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true +# mount sys reference: https://stackoverflow.com/questions/75808955/error-mounting-sys-kernel-debug-tracing-to-rootfs +sudo docker run -d -v /sys/:/sys/ --privileged --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true sudo docker cp /host/kyanos/kyanos alpine:/ sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/ sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/ diff --git a/testdata/test_add_cap_bpf.sh b/testdata/test_add_cap_bpf.sh index c295c9aa..940fda2b 100644 --- a/testdata/test_add_cap_bpf.sh +++ b/testdata/test_add_cap_bpf.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -set -ex +set -x CMD="$1" FILE_PREFIX="/tmp/kyanos" diff --git a/testdata/test_not_add_cap_bpf.sh b/testdata/test_not_add_cap_bpf.sh index e38a19f3..1a50b6ea 100644 --- a/testdata/test_not_add_cap_bpf.sh +++ b/testdata/test_not_add_cap_bpf.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -set -ex +set -x CMD="$1" FILE_PREFIX="/tmp/kyanos"