From d7bbea1c47f89e1aa04564a7b964dc9e90c4b00c Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 25 Nov 2024 07:19:02 +0000 Subject: [PATCH] Cleans up containers and shims on microk8s stop and snap removal The containerd shims and the containers are now removed in both classic and strict modes when stopping microk8s (microk8s stop) or when removing the snap. Note that ctr container delete has no ``--force`` flag. Adds additional assertion in the upgrade test, ensuring that the Pods / containers have been removed on snap removal. Adds additional test, ensuring that the Pods / containers have been removed when running microk8s stop. --- .github/workflows/build-snap.yml | 6 +++--- microk8s-resources/actions/common/utils.sh | 2 +- .../default-hooks/remove.d/90-containers | 8 ++------ microk8s-resources/wrappers/apiservice-kicker | 12 +++--------- .../wrappers/microk8s-stop.wrapper | 16 ++++++---------- snap/hooks/remove | 9 +++++++++ tests/requirements.txt | 1 + tests/test-simple.py | 16 ++++++++++++++++ tests/test-upgrade.py | 3 +++ tests/utils.py | 5 +++++ 10 files changed, 49 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index 3f26eb88ea..2c8a44d1ff 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -48,7 +48,7 @@ jobs: set -x sudo apt-get install python3-setuptools sudo pip3 install --upgrade pip - sudo pip3 install -U pytest sh + sudo pip3 install -U pytest sh psutil sudo apt-get -y install open-iscsi sudo systemctl enable iscsid - name: Fetch snap @@ -73,7 +73,7 @@ jobs: set -x sudo apt-get install python3-setuptools sudo pip3 install --upgrade pip - sudo pip3 install -U pytest sh + sudo pip3 install -U pytest sh psutil sudo apt-get -y install open-iscsi sudo systemctl enable iscsid - name: Fetch snap @@ -136,7 +136,7 @@ jobs: set -x sudo apt-get install python3-setuptools sudo pip3 install --upgrade pip - sudo pip3 install -U pytest sh + sudo pip3 install -U pytest sh psutil sudo apt-get -y install open-iscsi sudo systemctl enable iscsid - name: Fetch snap diff --git a/microk8s-resources/actions/common/utils.sh b/microk8s-resources/actions/common/utils.sh index 69179350b6..e3292f0a4c 100755 --- a/microk8s-resources/actions/common/utils.sh +++ b/microk8s-resources/actions/common/utils.sh @@ -988,7 +988,7 @@ remove_all_containers() { for container in $("${SNAP}/microk8s-ctr.wrapper" containers ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}') do - "${SNAP}/microk8s-ctr.wrapper" container delete --force $container &>/dev/null || true + "${SNAP}/microk8s-ctr.wrapper" container delete $container &>/dev/null || true done } diff --git a/microk8s-resources/default-hooks/remove.d/90-containers b/microk8s-resources/default-hooks/remove.d/90-containers index 620019bbb4..fcf510f12a 100755 --- a/microk8s-resources/default-hooks/remove.d/90-containers +++ b/microk8s-resources/default-hooks/remove.d/90-containers @@ -2,9 +2,5 @@ . "${SNAP}/actions/common/utils.sh" -if is_strict -then - remove_all_containers -else - kill_all_container_shims -fi +remove_all_containers +kill_all_container_shims diff --git a/microk8s-resources/wrappers/apiservice-kicker b/microk8s-resources/wrappers/apiservice-kicker index a29c38835a..fb353d51c3 100755 --- a/microk8s-resources/wrappers/apiservice-kicker +++ b/microk8s-resources/wrappers/apiservice-kicker @@ -65,15 +65,9 @@ do echo "cert change detected. Reconfiguring the kube-apiserver" rm -rf .srl snapctl stop microk8s.daemon-kubelite - if is_strict - then - remove_all_containers - snapctl restart microk8s.daemon-containerd - else - snapctl stop microk8s.daemon-containerd - kill_all_container_shims - snapctl start microk8s.daemon-containerd - fi + remove_all_containers + kill_all_container_shims + snapctl restart microk8s.daemon-containerd snapctl start microk8s.daemon-kubelite start_all_containers restart_attempt=$[$restart_attempt+1] diff --git a/microk8s-resources/wrappers/microk8s-stop.wrapper b/microk8s-resources/wrappers/microk8s-stop.wrapper index c942d59e92..4de540e513 100755 --- a/microk8s-resources/wrappers/microk8s-stop.wrapper +++ b/microk8s-resources/wrappers/microk8s-stop.wrapper @@ -40,13 +40,13 @@ while true; do esac done -stopcmd="run_with_sudo snap stop ${SNAP_NAME} --disable" +prefix_cmd="run_with_sudo snap" if is_strict then - stopcmd="snapctl stop microk8s.daemon-kubelite --disable" + prefix_cmd="snapctl" fi -$stopcmd +$prefix_cmd stop microk8s.daemon-kubelite --disable stop_status=$? if ! [ $stop_status -eq 0 ] @@ -54,12 +54,8 @@ then echo 'Failed to stop microk8s services. Check snapd logs with "journalctl -u snapd.service"' exit 1 else - if is_strict - then - remove_all_containers - snapctl stop microk8s --disable - else - kill_all_container_shims - fi + remove_all_containers + kill_all_container_shims + $prefix_cmd stop microk8s --disable run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock fi diff --git a/snap/hooks/remove b/snap/hooks/remove index f4e8932280..e0aa5d4c97 100755 --- a/snap/hooks/remove +++ b/snap/hooks/remove @@ -7,6 +7,15 @@ use_snap_env snapctl stop ${SNAP_NAME}.daemon-kubelite 2>&1 || true +# Temporarily start containerd so we can stop and kill all the microk8s containers. +snapctl start ${SNAP_NAME}.daemon-containerd 2>&1 || true +# wait for containerd to start. +sleep 5 + +# Remove any lingering containers and shims. +remove_all_containers +kill_all_container_shims + # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box. if test -L /var/lib/kubelet; then unlink /var/lib/kubelet || true diff --git a/tests/requirements.txt b/tests/requirements.txt index 93f8fdc657..dbe0f5b667 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,5 +7,6 @@ pyyaml sh jsonschema==4.0.0 pdbpp +psutil netifaces requests diff --git a/tests/test-simple.py b/tests/test-simple.py index deeb471778..757b9f1ef2 100644 --- a/tests/test-simple.py +++ b/tests/test-simple.py @@ -3,6 +3,8 @@ import requests import os.path +import utils + class TestSimple(object): def test_microk8s_nodes_ready(self): @@ -132,3 +134,17 @@ def test_microk8s_services_running(self): # Verify that all node services are running assert running_node_services == set(node_services), "Not all node services are running" + + def test_microk8s_stop_start(self): + coredns_procs = utils._get_process("coredns") + assert len(coredns_procs) > 0, "Expected to find a coredns process running." + + utils.run_until_success("/snap/bin/microk8s.stop", timeout_insec=180) + + new_coredns_procs = utils._get_process("coredns") + assert len(new_coredns_procs) == 0, "coredns found still running after microk8s stop." + + utils.run_until_success("/snap/bin/microk8s.start", timeout_insec=180) + + new_coredns_procs = utils._get_process("coredns") + assert len(new_coredns_procs) > 0, "Expected to find a new coredns process running." diff --git a/tests/test-upgrade.py b/tests/test-upgrade.py index 68dc7b0d05..279688f358 100644 --- a/tests/test-upgrade.py +++ b/tests/test-upgrade.py @@ -20,6 +20,7 @@ is_container, is_ipv6_configured, kubectl, + _get_process, ) upgrade_from = os.environ.get("UPGRADE_MICROK8S_FROM", "beta") @@ -157,3 +158,5 @@ def test_upgrade(self): if not is_container(): # On lxc umount docker overlay is not permitted. check_call("sudo snap remove microk8s".split()) + coredns_procs = _get_process("coredns") + assert len(coredns_procs) == 0, "Expected to have 0 coredns processes running." diff --git a/tests/utils.py b/tests/utils.py index 0dba44002f..b81c836d08 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,6 +3,7 @@ import time import yaml import platform +import psutil from subprocess import check_output, CalledProcessError, check_call @@ -270,3 +271,7 @@ def is_ipv6_configured(): return b"inet6" in output except CalledProcessError: return False + + +def _get_process(name): + return [p for p in psutil.process_iter() if name == p.name()]