Skip to content

Commit

Permalink
nixos/tests/incus: check hot-resize VM disks
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamcstephens
adamcstephens committed Oct 17, 2024
1 parent 7faea07 commit bd9355f
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 104 deletions.
4 changes: 1 addition & 3 deletions nixos/modules/virtualisation/incus.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ let
libnvidia-container
libxfs
lvm2
lxcfs
minio
minio-client
nftables
Expand All @@ -59,9 +60,6 @@ let
skopeo
umoci
]
++ lib.optionals (lib.versionAtLeast cfg.package.version "6.4.0") [
lxcfs
]
++ lib.optionals config.security.apparmor.enable [
apparmor-bin-utils
Expand Down
201 changes: 100 additions & 101 deletions nixos/tests/incus/container.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,107 +36,106 @@ in
networking.nftables.enable = true;
};

testScript = ''
def instance_is_up(_) -> bool:
status, _ = machine.execute("incus exec container --disable-stdin --force-interactive /run/current-system/sw/bin/systemctl -- is-system-running")
return status == 0
def set_container(config):
machine.succeed(f"incus config set container {config}")
machine.succeed("incus restart container")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
def check_sysctl(instance):
with subtest("systemd sysctl settings are applied"):
machine.succeed(f"incus exec {instance} -- systemctl status systemd-sysctl")
sysctl = machine.succeed(f"incus exec {instance} -- sysctl net.ipv4.ip_forward").strip().split(" ")[-1]
assert "1" == sysctl, f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"
machine.wait_for_unit("incus.service")
# no preseed should mean no service
machine.fail("systemctl status incus-preseed.service")
machine.succeed("incus admin init --minimal")
with subtest("Container image can be imported"):
machine.succeed("incus image import ${container-image-metadata} ${container-image-rootfs} --alias nixos")
with subtest("Container can be launched and managed"):
machine.succeed("incus launch nixos container")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
machine.succeed("echo true | incus exec container /run/current-system/sw/bin/bash -")
with subtest("Container mounts lxcfs overlays"):
machine.succeed("incus exec container mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'")
machine.succeed("incus exec container mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'")
with subtest("resource limits"):
with subtest("Container CPU limits can be managed"):
set_container("limits.cpu 1")
cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip()
assert cpuinfo == "1", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 1, got: {cpuinfo}"
set_container("limits.cpu 2")
cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip()
assert cpuinfo == "2", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 2, got: {cpuinfo}"
with subtest("Container memory limits can be managed"):
set_container("limits.memory 64MB")
meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip()
meminfo_bytes = " ".join(meminfo.split(' ')[-2:])
assert meminfo_bytes == "62500 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '62500 kB', got: '{meminfo_bytes}'"
set_container("limits.memory 128MB")
meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip()
meminfo_bytes = " ".join(meminfo.split(' ')[-2:])
assert meminfo_bytes == "125000 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '125000 kB', got: '{meminfo_bytes}'"
with subtest("lxc-generator"):
with subtest("lxc-container generator configures plain container"):
# reuse the existing container to save some time
machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
check_sysctl("container")
with subtest("lxc-container generator configures nested container"):
machine.execute("incus delete --force container")
machine.succeed("incus launch nixos container --config security.nesting=true")
with machine.nested("Waiting for instance to start and be usable"):
testScript = #python
''
def instance_is_up(_) -> bool:
status, _ = machine.execute("incus exec container --disable-stdin --force-interactive /run/current-system/sw/bin/systemctl -- is-system-running")
return status == 0
def set_container(config):
machine.succeed(f"incus config set container {config}")
machine.succeed("incus restart container")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
def check_sysctl(instance):
with subtest("systemd sysctl settings are applied"):
machine.succeed(f"incus exec {instance} -- systemctl status systemd-sysctl")
sysctl = machine.succeed(f"incus exec {instance} -- sysctl net.ipv4.ip_forward").strip().split(" ")[-1]
assert "1" == sysctl, f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"
machine.wait_for_unit("incus.service")
# no preseed should mean no service
machine.fail("systemctl status incus-preseed.service")
machine.succeed("incus admin init --minimal")
with subtest("Container image can be imported"):
machine.succeed("incus image import ${container-image-metadata} ${container-image-rootfs} --alias nixos")
with subtest("Container can be launched and managed"):
machine.succeed("incus launch nixos container")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
machine.succeed("echo true | incus exec container /run/current-system/sw/bin/bash -")
with subtest("Container mounts lxcfs overlays"):
machine.succeed("incus exec container mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'")
machine.succeed("incus exec container mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'")
with subtest("resource limits"):
with subtest("Container CPU limits can be managed"):
set_container("limits.cpu 1")
cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip()
assert cpuinfo == "1", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 1, got: {cpuinfo}"
set_container("limits.cpu 2")
cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip()
assert cpuinfo == "2", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 2, got: {cpuinfo}"
with subtest("Container memory limits can be managed"):
set_container("limits.memory 64MB")
meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip()
meminfo_bytes = " ".join(meminfo.split(' ')[-2:])
assert meminfo_bytes == "62500 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '62500 kB', got: '{meminfo_bytes}'"
set_container("limits.memory 128MB")
meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip()
meminfo_bytes = " ".join(meminfo.split(' ')[-2:])
assert meminfo_bytes == "125000 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '125000 kB', got: '{meminfo_bytes}'"
with subtest("lxc-generator"):
with subtest("lxc-container generator configures plain container"):
# reuse the existing container to save some time
machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
check_sysctl("container")
with subtest("lxc-container generator configures nested container"):
machine.execute("incus delete --force container")
machine.succeed("incus launch nixos container --config security.nesting=true")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
machine.fail("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
target = machine.succeed("incus exec container readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"
check_sysctl("container")
with subtest("lxc-container generator configures privileged container"):
machine.execute("incus delete --force container")
machine.succeed("incus launch nixos container --config security.privileged=true")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
check_sysctl("container")
with subtest("supports per-instance lxcfs"):
machine.fail("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'")
machine.succeed("incus config set instances.lxcfs.per_instance=true")
machine.succeed("incus restart container")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
machine.fail("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
target = machine.succeed("incus exec container readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"
check_sysctl("container")
with subtest("lxc-container generator configures privileged container"):
machine.execute("incus delete --force container")
machine.succeed("incus launch nixos container --config security.privileged=true")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
check_sysctl("container")
'' + lib.optionalString (lib.versionAtLeast incus.version "6.4.0") ''
with subtest("supports per-instance lxcfs"):
machine.fail("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'")
machine.succeed("incus config set instances.lxcfs.per_instance=true")
machine.succeed("incus restart container")
with machine.nested("Waiting for instance to start and be usable"):
retry(instance_is_up)
machine.succeed("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'")
'' + ''
with subtest("softDaemonRestart"):
with subtest("Instance remains running when softDaemonRestart is enabled and services is stopped"):
pid = machine.succeed("incus info container | grep 'PID'").split(":")[1].strip()
machine.succeed(f"ps {pid}")
machine.succeed("systemctl stop incus")
machine.succeed(f"ps {pid}")
machine.succeed("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'")
with subtest("softDaemonRestart"):
with subtest("Instance remains running when softDaemonRestart is enabled and services is stopped"):
pid = machine.succeed("incus info container | grep 'PID'").split(":")[1].strip()
machine.succeed(f"ps {pid}")
machine.succeed("systemctl stop incus")
machine.succeed(f"ps {pid}")
'';
})
18 changes: 18 additions & 0 deletions nixos/tests/incus/virtual-machine.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ in
};

testScript = ''
import json
def instance_is_up(_) -> bool:
status, _ = machine.execute("incus exec ${instance-name} --disable-stdin --force-interactive /run/current-system/sw/bin/systemctl -- is-system-running")
return status == 0
Expand Down Expand Up @@ -79,6 +81,22 @@ in
count = int(machine.succeed("incus exec ${instance-name} -- nproc").strip())
assert count == 2, f"Wrong number of CPUs reported, want: 2, got: {count}"
def get_root_disk_size():
lsblk = json.loads(machine.succeed("incus exec ${instance-name} -- lsblk /dev/sda --json"))
for blk in lsblk["blockdevices"]:
if blk["name"] == "sda":
size = blk["size"]
break
return size
with subtest("guest supports live disk resize"):
size = get_root_disk_size()
assert size == "10G", f"Wrong root disk size reported, want: 10G, got: {size}"
machine.succeed("incus config device override ${instance-name} root size=20GiB")
size = get_root_disk_size()
assert size == "20G", f"Wrong root disk size reported, want: 20G, got: {size}"
with subtest("Instance remains running when softDaemonRestart is enabled and services is stopped"):
pid = machine.succeed("incus info ${instance-name} | grep 'PID'").split(":")[1].strip()
machine.succeed(f"ps {pid}")
Expand Down

0 comments on commit bd9355f

Please sign in to comment.