From b1e8c44050b08bac54f07ed5be43562b16ef8ec7 Mon Sep 17 00:00:00 2001
From: Kornilios Kourtis <kornilios@isovalent.com>
Date: Fri, 9 Aug 2024 15:40:13 +0200
Subject: [PATCH] cgroups: deal with crun subgroups
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

crun, an OCI container runtime used by cri-o breaks pod association for
tetragon by using placing processes in a cgroup below the cgroup specified by
the OCI spec:
https://github.com/containers/crun/blob/main/crun.1.md#runocisystemdsubgroupsubgroup.

With the introduction of cgidmap, this commit can finally deal with this issue
by scanning the cgroup directory for children directories and, if it finds one,
use the cgroup id of the child.

A better solution would be to allow for multiple cgroup ids for each container,
but this is left as a followup.

The commit includes a script for testing this issue using minikube. Becaues
minikube uses an older version of crun, we need to install it.

The steps for reproducing this are:
   minikube start --driver=kvm2 --container-runtime=crio --force-systemd=true
   ./scripts/minikube-install-crun.sh

Running tetragon without cgidmap, we observe events without pod association:
  🚀 process minikube /usr/bin/ls
  💥 exit    minikube /usr/bin/ls  0

By installing the runtime hooks:
 ./scripts/minikube-install-hook.sh

And runing tetragon with cgidmap (and nri) using --enable-cri --enable-cgidmap,
we observe pod association for both old and new pods:

🚀 process default/test /usr/bin/ls
💥 exit    default/test /usr/bin/ls  0

Signed-off-by: Kornilios Kourtis <kornilios@isovalent.com>
---
 .../scripts/minikube-install-crun.sh          | 70 +++++++++++++++++++
 pkg/cgidmap/cri.go                            |  2 +-
 pkg/cgroups/cgroups.go                        | 42 +++++++++++
 pkg/policyfilter/cgroupid.go                  |  2 +-
 pkg/rthooks/args.go                           |  2 +-
 5 files changed, 115 insertions(+), 3 deletions(-)
 create mode 100755 contrib/tetragon-rthooks/scripts/minikube-install-crun.sh

diff --git a/contrib/tetragon-rthooks/scripts/minikube-install-crun.sh b/contrib/tetragon-rthooks/scripts/minikube-install-crun.sh
new file mode 100755
index 00000000000..f487dd8c8a2
--- /dev/null
+++ b/contrib/tetragon-rthooks/scripts/minikube-install-crun.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+# vim:set noet ci pi ts=4 sw=4
+
+set -o pipefail
+set -e
+
+if [ "$1" != "install" ]; then
+	SCRIPTPATH=$(dirname "$0")
+	source ${SCRIPTPATH}/helpers
+
+	runtime=$(detect_runtime)
+	if [ "$runtime" != "crio" ]; then
+		echo "crio not installed, bailing out"
+		exit 1
+	fi
+
+	name=$(basename "$0")
+	minikube cp $0 /tmp/$name
+	minikube ssh sudo chmod +x /tmp/$name
+	minikube ssh sudo /tmp/$name install
+	exit 0
+fi
+
+set -x
+
+echo "Running inside minikube: $(uname -a)"
+crio_v=$(crio --version | sed -ne 's/^Version:[[:space:]]\+\(.\+\)/\1/p')
+echo "crio version: $crio_v"
+crun_v=$(crun --version | sed -ne 's/^crun version[[:space:]]\+\(.\+\)/\1/p')
+echo "old crun version: $crun_v"
+
+# cleanup everything
+systemctl stop kubelet
+crictl ps -a -q | xargs crictl stop
+crictl ps -a -q | xargs crictl rm
+crictl pods -q | xargs crictl stopp
+crictl pods -q | xargs crictl rmp
+systemctl stop crio
+
+cd /tmp
+tarball=cri-o.amd64.v${crio_v}.tar.gz
+if [ -f "${tarball}" ]; then
+	echo "tarball ${tarball} exists, skipping download"
+else
+	curl -sOL -C - https://storage.googleapis.com/cri-o/artifacts/${tarball}
+fi
+rm -rf cri-o
+tar zxf $tarball
+cd cri-o
+cp ./bin/crio-{conmon,conmonrs,crun} /usr/bin
+crio_crun_v=$(crio-crun --version | sed -ne 's/^crun version[[:space:]]\+\(.\+\)/\1/p')
+echo "new crun version: $crio_crun_v"
+
+fname=$(mktemp -t crio-crun-conf.XXXXX)
+cat >$fname <<EOF
+[crio.runtime]
+default_runtime = "crun"
+
+[crio.runtime.runtimes.crun]
+runtime_path = "/usr/bin/crio-crun"
+monitor_path = "/usr/bin/crio-conmon"
+allowed_annotations = [
+    "io.containers.trace-syscall",
+]
+EOF
+chmod go+r ${fname}
+chown root:root ${fname}
+cp $fname /etc/crio/crio.conf.d/10-crun.conf
+systemctl start crio
+systemctl start kubelet
diff --git a/pkg/cgidmap/cri.go b/pkg/cgidmap/cri.go
index aadd428b831..faadf1c0c1b 100644
--- a/pkg/cgidmap/cri.go
+++ b/pkg/cgidmap/cri.go
@@ -85,7 +85,7 @@ func criResolve(m Map, id unmappedID) error {
 	}
 
 	path := filepath.Join(cgRoot, cgPath)
-	cgID, err := cgroups.GetCgroupIdFromPath(path)
+	cgID, err := cgroups.GetCgroupIDFromSubCgroup(path)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go
index 6ff9efba486..7beaf2932e9 100644
--- a/pkg/cgroups/cgroups.go
+++ b/pkg/cgroups/cgroups.go
@@ -822,3 +822,45 @@ func CgroupIDFromPID(pid uint32) (uint64, error) {
 
 	return cgID, nil
 }
+
+// GetCgroupIDFromSubCgroup deals with some idiosyncrancies of container runtimes
+//
+// Typically, the container processes run in the cgroup path specified in the OCI spec under
+// cgroupsPath. crun, however, is an exception because it uses another directory (called subgroup)
+// under the cgroupsPath:
+// https://github.com/containers/crun/blob/main/crun.1.md#runocisystemdsubgroupsubgroup.
+//
+// This function deals with this by checking for a child directory. If it finds one (and only one)
+// it uses the cgroup id from the child.
+func GetCgroupIDFromSubCgroup(p string) (uint64, error) {
+
+	getSingleDirChild := func() string {
+		var ret string
+		dentries, err := os.ReadDir(p)
+		if err != nil {
+			return ""
+		}
+		for _, dentry := range dentries {
+			if !dentry.IsDir() {
+				continue
+			}
+
+			if ret == "" {
+				ret = dentry.Name()
+			} else {
+				// NB: there are more than one directories :( nothing reasonable we
+				// can do at this point bail out
+				return ""
+			}
+		}
+
+		return ret
+	}
+
+	child := getSingleDirChild()
+	if child != "" {
+		p = filepath.Join(p, child)
+	}
+
+	return GetCgroupIdFromPath(p)
+}
diff --git a/pkg/policyfilter/cgroupid.go b/pkg/policyfilter/cgroupid.go
index 9659056e828..24aa64f313d 100644
--- a/pkg/policyfilter/cgroupid.go
+++ b/pkg/policyfilter/cgroupid.go
@@ -31,6 +31,6 @@ func (s *cgfsFinder) findCgroupID(podID PodID, containerID string) (CgroupID, er
 	} else if err != nil {
 		return CgroupID(0), err
 	}
-	cgid, err := cgroups.GetCgroupIdFromPath(path)
+	cgid, err := cgroups.GetCgroupIDFromSubCgroup(path)
 	return CgroupID(cgid), err
 }
diff --git a/pkg/rthooks/args.go b/pkg/rthooks/args.go
index 591c5f1d922..8adaeb57802 100644
--- a/pkg/rthooks/args.go
+++ b/pkg/rthooks/args.go
@@ -46,7 +46,7 @@ func (arg *CreateContainerArg) CgroupID() (uint64, error) {
 	}
 
 	path := filepath.Join(cgRoot, cgPath)
-	cgID, err := cgroups.GetCgroupIdFromPath(path)
+	cgID, err := cgroups.GetCgroupIDFromSubCgroup(path)
 	if err != nil {
 		return 0, err
 	}