Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Overlay handling with fuse-overlayfs #1062

Merged
merged 36 commits into from
Sep 20, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
be28a5f
feat: Implement FUSE-based overlay mount for containerexec and runexec
Jul 2, 2024
0facbcf
feat: Support FUSE-based overlay mount for benchexec
Jul 2, 2024
85f02ca
Only if the kernel overlay fails, try using fuse-overlayfs
Jul 7, 2024
69e929e
refactor: added user-visible messages and some refactoring
younghojan Jul 10, 2024
9818716
feat: Clear ambient capabilities in drop_capabilities() and add const…
Aug 4, 2024
3f69c41
Merge branch 'main' into gsoc-overlay-handling-with-fuse-overlayfs-dev
younghojan Aug 4, 2024
4ee99a3
Merge branch 'main' into gsoc-overlay-handling-with-fuse-overlayfs-dev
PhilippWendler Aug 6, 2024
a699b2f
chore: Fix bug in cap_permitted_to_ambient function
Aug 11, 2024
8006c20
fix: Use single fusermount for all fuse-based overlays, and avoid mix…
Aug 11, 2024
195e4d0
chore: Add functions and extracted some code into functions, add comm…
Aug 13, 2024
00f9cb8
chore: Refactor some functions related to fuse-based overlay mounts a…
Aug 13, 2024
328aad4
chore: Refactor functions related to fuse-based overlay mounts and im…
Aug 13, 2024
de86749
chore: Refactor functions related to fuse-based overlay mounts and im…
Aug 14, 2024
a308c46
chore: Replace f-string in logging.debug with %s formatting
Aug 15, 2024
b941329
Add fuse-overlayfs to our recommended dependencies
PhilippWendler Aug 16, 2024
2529120
Update documentation on kernel overlayfs vs. fuse-overlayfs
PhilippWendler Aug 16, 2024
dde34ea
test: Add tests for checking fuse-overlayfs functionality and triple-…
younghojan Aug 17, 2024
b1a02d6
fix: Specify stdin=subprocess.DEVNULL when launching the fuse-overlay…
Aug 22, 2024
1f6d696
feat: Check if fuse-overlayfs meets the minimum version requirement, …
Aug 26, 2024
ba6bb91
Merge branch 'main' into gsoc-overlay-handling-with-fuse-overlayfs-dev
PhilippWendler Aug 26, 2024
dc482b2
fix: fix issue of checking for fuse-overlayfs functionality outside o…
Aug 28, 2024
e0aec8c
chore: Refactor and improve test_triple_nested_runexec
Aug 29, 2024
e0833b3
chore: Refactor fuse-overlayfs setup and error handling
Aug 29, 2024
147b4e2
Merge 'main' into gsoc-overlay-handling-with-fuse-overlayfs-dev
PhilippWendler Sep 2, 2024
b63db00
Refactor and improve fuse-overlay related tests
Sep 2, 2024
38a0508
Omit test_triple_nested_runexec when coverage testing
Sep 4, 2024
5d2a349
Refactor COV_CORE_SOURCE environment variable handling
Sep 4, 2024
a8a3516
Safely encode string for fuse-overlayfs paths
Sep 5, 2024
34f57f1
Refactor determine_directory_mode function for fuse-overlayfs compati…
Sep 5, 2024
2fd26ff
Refactor file handling in test_runexecutor.py for better readability
Sep 5, 2024
88db419
Refactor overlay mount error handling for better compatibility
Sep 5, 2024
2f9d52e
Fix typo
Sep 15, 2024
1c49af2
Refactor handling of COV_CORE_SOURCE environment variable in TestRunE…
Sep 15, 2024
ea92000
Change internal paths used for fuse-overlayfs mounts
PhilippWendler Sep 19, 2024
ec11b7f
Add logging about why fuse-overlayfs is used
PhilippWendler Sep 19, 2024
33249f1
Detect and error out if temp is not hidden and we use fuse-overlayfs
PhilippWendler Sep 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 92 additions & 17 deletions benchexec/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import socket
import struct
import sys
import subprocess

from benchexec import libc
from benchexec import seccomp
Expand All @@ -33,6 +34,7 @@
"get_mount_points",
"remount_with_additional_flags",
"make_overlay_mount",
"make_fuse_overlay_mount",
"mount_proc",
"make_bind_mount",
"get_my_pid_from_procfs",
Expand Down Expand Up @@ -507,29 +509,45 @@ def duplicate_mount_hierarchy(mount_base, temp_base, work_base, dir_modes):
work_path = work_base + b"/" + str(overlay_count).encode()
os.makedirs(temp_path, exist_ok=True)
os.makedirs(work_path, exist_ok=True)
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved
try:
# Previous mount in this place not needed if replaced with overlay dir.
libc.umount(mount_path)
except OSError as e:
logging.debug(e)
if os.path.ismount(mount_path):
try:
# Previous mount in this place not needed if replaced with overlay dir.
libc.umount(mount_path)
except OSError as e:
logging.debug(e)
try:
make_overlay_mount(mount_path, mountpoint, temp_path, work_path)
except OSError as e:
mp = mountpoint.decode()
raise OSError(
e.errno,
f"Creating overlay mount for '{mp}' failed: {os.strerror(e.errno)}. "
f"Please use other directory modes, "
f"for example '--read-only-dir {util.escape_string_shell(mp)}'.",
)
# Resort to fuse-overlayfs if kernel overlayfs is not available.
fuse = util.find_executable2("fuse-overlayfs")
if fuse:
logging.debug(
"Cannot use kernel overlay for %s: %s. "
"Trying to use fuse-overlayfs instead.",
mountpoint.decode(),
e,
)
cap_permitted_to_ambient()
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved
make_fuse_overlay_mount(
fuse, mount_path, mountpoint, temp_path, work_path
)
else:
mp = mountpoint.decode()
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved
raise OSError(
e.errno,
f"Creating overlay mount for '{mp}' failed: {os.strerror(e.errno)}. "
f"Please use other directory modes, "
f"for example '--read-only-dir {util.escape_string_shell(mp)}'.",
)

elif mode == DIR_HIDDEN:
os.makedirs(temp_path, exist_ok=True)
try:
# Previous mount in this place not needed if replaced with hidden dir.
libc.umount(mount_path)
except OSError as e:
logging.debug(e)
if os.path.ismount(mount_path):
try:
# Previous mount in this place not needed if replaced with hidden dir.
libc.umount(mount_path)
except OSError as e:
logging.debug(e)
make_bind_mount(temp_path, mount_path)

elif mode == DIR_READ_ONLY:
Expand Down Expand Up @@ -722,6 +740,37 @@ def escape(s):
)


def make_fuse_overlay_mount(exe, mount, lower, upper, work):
logging.debug(
"Creating overlay mount: target=%s, lower=%s, upper=%s, work=%s",
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved
mount,
lower,
upper,
work,
)

def escape(s):
return s.replace(b"\\", rb"\\").replace(b":", rb"\:").replace(b",", rb"\,")
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved

cmd = (
exe,
b"-o",
b"lowerdir="
+ escape(lower)
+ b",upperdir="
+ escape(upper)
+ b",workdir="
+ escape(work),
escape(mount),
)

try:
subprocess.run(args=cmd, check=True)
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved
except subprocess.CalledProcessError as e:
logging.error("Error executing command: %s", e)
sys.exit(1)


def mount_proc(container_system_config):
"""Mount the /proc filesystem.
@param container_system_config: Whether to mount container-specific files in /proc
Expand Down Expand Up @@ -812,6 +861,32 @@ def drop_capabilities(keep=[]):
)


def cap_permitted_to_ambient():
"""
Python version of util-linux/lib/caputils.c: cap_permitted_to_ambient()
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved
"""

header = libc.CapHeader(libc.LINUX_CAPABILITY_VERSION_3, 0)
data = (libc.CapData * libc.LINUX_CAPABILITY_U32S_3)()

libc.capget(header, data)

data[0].inheritable = data[0].permitted
data[1].inheritable = data[1].permitted

libc.capset(header, data)

effective = (data[1].effective << 32) | data[0].effective
for cap in range(64):
if cap > int(util.try_read_file("/proc/sys/kernel/cap_last_cap")):
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved
continue

if effective & (1 << cap):
libc.prctl(
47, 2, cap, 0, 0
) # 47 = PR_CAP_AMBIENT, 2 = PR_CAP_AMBIENT_RAISE
PhilippWendler marked this conversation as resolved.
Show resolved Hide resolved


_FORBIDDEN_SYSCALLS = [
# Kernel keyrings are not namespaced before Linux 5.2.
b"add_key",
Expand Down
7 changes: 7 additions & 0 deletions benchexec/containerexecutor.py
Original file line number Diff line number Diff line change
Expand Up @@ -684,6 +684,13 @@ def child():
container.get_my_pid_from_procfs(),
)

# Ensure that capabilities granted in the user namespace are
# preserved in the child process.
#
# TODO: We still don't know which capabilities are necessary,
# and the rest could be dropped.
container.cap_permitted_to_ambient()

# Put all received signals on hold until we handle them later.
container.block_all_signals()

Expand Down
5 changes: 4 additions & 1 deletion benchexec/containerized_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,6 @@ def _init_container(

# Container config
container.setup_user_mapping(os.getpid(), uid, gid)
_setup_container_filesystem(temp_dir, dir_modes, container_system_config)
if container_system_config:
socket.sethostname(container.CONTAINER_HOSTNAME)
if not network_access:
Expand All @@ -217,6 +216,10 @@ def _init_container(
os.waitpid(pid, 0)
os._exit(0)

# We setup the container's filesystem in the child process.
# Delaying this until after the fork can avoid "Transport endpoint not connected" issue.
_setup_container_filesystem(temp_dir, dir_modes, container_system_config)

# Finalize container setup in child
container.mount_proc(container_system_config) # only possible in child
container.drop_capabilities()
Expand Down
10 changes: 10 additions & 0 deletions benchexec/libc.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,14 +184,24 @@ class CapData(_ctypes.Structure):
_ctypes.POINTER(CapData * 2),
]

capget = _libc.capget
"""Get the capabilities of the current thread."""
capget.errcheck = _check_errno
capget.argtypes = [
_ctypes.POINTER(CapHeader),
_ctypes.POINTER(CapData * 2),
]

LINUX_CAPABILITY_VERSION_3 = 0x20080522 # /usr/include/linux/capability.h
LINUX_CAPABILITY_U32S_3 = 2 # /usr/include/linux/capability.h
CAP_SYS_ADMIN = 21 # /usr/include/linux/capability.h

prctl = _libc.prctl
"""Modify options of processes: http://man7.org/linux/man-pages/man2/prctl.2.html"""
prctl.errcheck = _check_errno
prctl.argtypes = [c_int, c_ulong, c_ulong, c_ulong, c_ulong]


# /usr/include/linux/prctl.h
PR_SET_DUMPABLE = 4
PR_GET_SECCOMP = 21
Expand Down