From bd9355f00a7244ff2e7104d6740eee4894b3aa5d Mon Sep 17 00:00:00 2001 From: Adam Stephens Date: Sun, 8 Sep 2024 11:39:05 -0400 Subject: [PATCH] nixos/tests/incus: check hot-resize VM disks --- nixos/modules/virtualisation/incus.nix | 4 +- nixos/tests/incus/container.nix | 201 ++++++++++++------------- nixos/tests/incus/virtual-machine.nix | 18 +++ 3 files changed, 119 insertions(+), 104 deletions(-) diff --git a/nixos/modules/virtualisation/incus.nix b/nixos/modules/virtualisation/incus.nix index 5a199db86eb8a..407862ff9dd8b 100644 --- a/nixos/modules/virtualisation/incus.nix +++ b/nixos/modules/virtualisation/incus.nix @@ -38,6 +38,7 @@ let libnvidia-container libxfs lvm2 + lxcfs minio minio-client nftables @@ -59,9 +60,6 @@ let skopeo umoci ] - ++ lib.optionals (lib.versionAtLeast cfg.package.version "6.4.0") [ - lxcfs - ] ++ lib.optionals config.security.apparmor.enable [ apparmor-bin-utils diff --git a/nixos/tests/incus/container.nix b/nixos/tests/incus/container.nix index 8cac333d9ee65..7b9029f197561 100644 --- a/nixos/tests/incus/container.nix +++ b/nixos/tests/incus/container.nix @@ -36,107 +36,106 @@ in networking.nftables.enable = true; }; - testScript = '' - def instance_is_up(_) -> bool: - status, _ = machine.execute("incus exec container --disable-stdin --force-interactive /run/current-system/sw/bin/systemctl -- is-system-running") - return status == 0 - - def set_container(config): - machine.succeed(f"incus config set container {config}") - machine.succeed("incus restart container") - with machine.nested("Waiting for instance to start and be usable"): - retry(instance_is_up) - - def check_sysctl(instance): - with subtest("systemd sysctl settings are applied"): - machine.succeed(f"incus exec {instance} -- systemctl status systemd-sysctl") - sysctl = machine.succeed(f"incus exec {instance} -- sysctl net.ipv4.ip_forward").strip().split(" ")[-1] - assert "1" == sysctl, f"systemd-sysctl configuration not correctly applied, {sysctl} != 1" - - machine.wait_for_unit("incus.service") - - # no preseed should mean no service - machine.fail("systemctl status incus-preseed.service") - - machine.succeed("incus admin init --minimal") - - with subtest("Container image can be imported"): - machine.succeed("incus image import ${container-image-metadata} ${container-image-rootfs} --alias nixos") - - with subtest("Container can be launched and managed"): - machine.succeed("incus launch nixos container") - with machine.nested("Waiting for instance to start and be usable"): - retry(instance_is_up) - machine.succeed("echo true | incus exec container /run/current-system/sw/bin/bash -") - - with subtest("Container mounts lxcfs overlays"): - machine.succeed("incus exec container mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'") - machine.succeed("incus exec container mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'") - - with subtest("resource limits"): - with subtest("Container CPU limits can be managed"): - set_container("limits.cpu 1") - cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip() - assert cpuinfo == "1", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 1, got: {cpuinfo}" - - set_container("limits.cpu 2") - cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip() - assert cpuinfo == "2", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 2, got: {cpuinfo}" - - with subtest("Container memory limits can be managed"): - set_container("limits.memory 64MB") - meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip() - meminfo_bytes = " ".join(meminfo.split(' ')[-2:]) - assert meminfo_bytes == "62500 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '62500 kB', got: '{meminfo_bytes}'" - - set_container("limits.memory 128MB") - meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip() - meminfo_bytes = " ".join(meminfo.split(' ')[-2:]) - assert meminfo_bytes == "125000 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '125000 kB', got: '{meminfo_bytes}'" - - with subtest("lxc-generator"): - with subtest("lxc-container generator configures plain container"): - # reuse the existing container to save some time - machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") - check_sysctl("container") - - with subtest("lxc-container generator configures nested container"): - machine.execute("incus delete --force container") - machine.succeed("incus launch nixos container --config security.nesting=true") - with machine.nested("Waiting for instance to start and be usable"): + testScript = #python + '' + def instance_is_up(_) -> bool: + status, _ = machine.execute("incus exec container --disable-stdin --force-interactive /run/current-system/sw/bin/systemctl -- is-system-running") + return status == 0 + + def set_container(config): + machine.succeed(f"incus config set container {config}") + machine.succeed("incus restart container") + with machine.nested("Waiting for instance to start and be usable"): + retry(instance_is_up) + + def check_sysctl(instance): + with subtest("systemd sysctl settings are applied"): + machine.succeed(f"incus exec {instance} -- systemctl status systemd-sysctl") + sysctl = machine.succeed(f"incus exec {instance} -- sysctl net.ipv4.ip_forward").strip().split(" ")[-1] + assert "1" == sysctl, f"systemd-sysctl configuration not correctly applied, {sysctl} != 1" + + machine.wait_for_unit("incus.service") + + # no preseed should mean no service + machine.fail("systemctl status incus-preseed.service") + + machine.succeed("incus admin init --minimal") + + with subtest("Container image can be imported"): + machine.succeed("incus image import ${container-image-metadata} ${container-image-rootfs} --alias nixos") + + with subtest("Container can be launched and managed"): + machine.succeed("incus launch nixos container") + with machine.nested("Waiting for instance to start and be usable"): + retry(instance_is_up) + machine.succeed("echo true | incus exec container /run/current-system/sw/bin/bash -") + + with subtest("Container mounts lxcfs overlays"): + machine.succeed("incus exec container mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'") + machine.succeed("incus exec container mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'") + + with subtest("resource limits"): + with subtest("Container CPU limits can be managed"): + set_container("limits.cpu 1") + cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip() + assert cpuinfo == "1", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 1, got: {cpuinfo}" + + set_container("limits.cpu 2") + cpuinfo = machine.succeed("incus exec container grep -- -c ^processor /proc/cpuinfo").strip() + assert cpuinfo == "2", f"Wrong number of CPUs reported from /proc/cpuinfo, want: 2, got: {cpuinfo}" + + with subtest("Container memory limits can be managed"): + set_container("limits.memory 64MB") + meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip() + meminfo_bytes = " ".join(meminfo.split(' ')[-2:]) + assert meminfo_bytes == "62500 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '62500 kB', got: '{meminfo_bytes}'" + + set_container("limits.memory 128MB") + meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip() + meminfo_bytes = " ".join(meminfo.split(' ')[-2:]) + assert meminfo_bytes == "125000 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '125000 kB', got: '{meminfo_bytes}'" + + with subtest("lxc-generator"): + with subtest("lxc-container generator configures plain container"): + # reuse the existing container to save some time + machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") + check_sysctl("container") + + with subtest("lxc-container generator configures nested container"): + machine.execute("incus delete --force container") + machine.succeed("incus launch nixos container --config security.nesting=true") + with machine.nested("Waiting for instance to start and be usable"): + retry(instance_is_up) + + machine.fail("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") + target = machine.succeed("incus exec container readlink -- -f /run/systemd/system/systemd-binfmt.service").strip() + assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service" + + check_sysctl("container") + + with subtest("lxc-container generator configures privileged container"): + machine.execute("incus delete --force container") + machine.succeed("incus launch nixos container --config security.privileged=true") + with machine.nested("Waiting for instance to start and be usable"): + retry(instance_is_up) + + machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") + + check_sysctl("container") + + with subtest("supports per-instance lxcfs"): + machine.fail("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'") + machine.succeed("incus config set instances.lxcfs.per_instance=true") + machine.succeed("incus restart container") + with machine.nested("Waiting for instance to start and be usable"): retry(instance_is_up) - - machine.fail("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") - target = machine.succeed("incus exec container readlink -- -f /run/systemd/system/systemd-binfmt.service").strip() - assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service" - - check_sysctl("container") - - with subtest("lxc-container generator configures privileged container"): - machine.execute("incus delete --force container") - machine.succeed("incus launch nixos container --config security.privileged=true") - with machine.nested("Waiting for instance to start and be usable"): - retry(instance_is_up) - - machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") - - check_sysctl("container") - - '' + lib.optionalString (lib.versionAtLeast incus.version "6.4.0") '' - with subtest("supports per-instance lxcfs"): - machine.fail("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'") - machine.succeed("incus config set instances.lxcfs.per_instance=true") - machine.succeed("incus restart container") - with machine.nested("Waiting for instance to start and be usable"): - retry(instance_is_up) - machine.succeed("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'") - - '' + '' - with subtest("softDaemonRestart"): - with subtest("Instance remains running when softDaemonRestart is enabled and services is stopped"): - pid = machine.succeed("incus info container | grep 'PID'").split(":")[1].strip() - machine.succeed(f"ps {pid}") - machine.succeed("systemctl stop incus") - machine.succeed(f"ps {pid}") + machine.succeed("pgrep -a lxcfs | grep 'incus/devices/container/lxcfs'") + + with subtest("softDaemonRestart"): + with subtest("Instance remains running when softDaemonRestart is enabled and services is stopped"): + pid = machine.succeed("incus info container | grep 'PID'").split(":")[1].strip() + machine.succeed(f"ps {pid}") + machine.succeed("systemctl stop incus") + machine.succeed(f"ps {pid}") ''; }) diff --git a/nixos/tests/incus/virtual-machine.nix b/nixos/tests/incus/virtual-machine.nix index f5ac4c8eee1f2..52a1e00983c5e 100644 --- a/nixos/tests/incus/virtual-machine.nix +++ b/nixos/tests/incus/virtual-machine.nix @@ -42,6 +42,8 @@ in }; testScript = '' + import json + def instance_is_up(_) -> bool: status, _ = machine.execute("incus exec ${instance-name} --disable-stdin --force-interactive /run/current-system/sw/bin/systemctl -- is-system-running") return status == 0 @@ -79,6 +81,22 @@ in count = int(machine.succeed("incus exec ${instance-name} -- nproc").strip()) assert count == 2, f"Wrong number of CPUs reported, want: 2, got: {count}" + def get_root_disk_size(): + lsblk = json.loads(machine.succeed("incus exec ${instance-name} -- lsblk /dev/sda --json")) + for blk in lsblk["blockdevices"]: + if blk["name"] == "sda": + size = blk["size"] + break + return size + + with subtest("guest supports live disk resize"): + size = get_root_disk_size() + assert size == "10G", f"Wrong root disk size reported, want: 10G, got: {size}" + + machine.succeed("incus config device override ${instance-name} root size=20GiB") + size = get_root_disk_size() + assert size == "20G", f"Wrong root disk size reported, want: 20G, got: {size}" + with subtest("Instance remains running when softDaemonRestart is enabled and services is stopped"): pid = machine.succeed("incus info ${instance-name} | grep 'PID'").split(":")[1].strip() machine.succeed(f"ps {pid}")