diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index 3f26eb88ea..2c8a44d1ff 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -48,7 +48,7 @@ jobs: set -x sudo apt-get install python3-setuptools sudo pip3 install --upgrade pip - sudo pip3 install -U pytest sh + sudo pip3 install -U pytest sh psutil sudo apt-get -y install open-iscsi sudo systemctl enable iscsid - name: Fetch snap @@ -73,7 +73,7 @@ jobs: set -x sudo apt-get install python3-setuptools sudo pip3 install --upgrade pip - sudo pip3 install -U pytest sh + sudo pip3 install -U pytest sh psutil sudo apt-get -y install open-iscsi sudo systemctl enable iscsid - name: Fetch snap @@ -136,7 +136,7 @@ jobs: set -x sudo apt-get install python3-setuptools sudo pip3 install --upgrade pip - sudo pip3 install -U pytest sh + sudo pip3 install -U pytest sh psutil sudo apt-get -y install open-iscsi sudo systemctl enable iscsid - name: Fetch snap diff --git a/microk8s-resources/actions/common/utils.sh b/microk8s-resources/actions/common/utils.sh index 69179350b6..e3292f0a4c 100755 --- a/microk8s-resources/actions/common/utils.sh +++ b/microk8s-resources/actions/common/utils.sh @@ -988,7 +988,7 @@ remove_all_containers() { for container in $("${SNAP}/microk8s-ctr.wrapper" containers ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}') do - "${SNAP}/microk8s-ctr.wrapper" container delete --force $container &>/dev/null || true + "${SNAP}/microk8s-ctr.wrapper" container delete $container &>/dev/null || true done } diff --git a/microk8s-resources/default-hooks/remove.d/90-containers b/microk8s-resources/default-hooks/remove.d/90-containers index 620019bbb4..fcf510f12a 100755 --- a/microk8s-resources/default-hooks/remove.d/90-containers +++ b/microk8s-resources/default-hooks/remove.d/90-containers @@ -2,9 +2,5 @@ . "${SNAP}/actions/common/utils.sh" -if is_strict -then - remove_all_containers -else - kill_all_container_shims -fi +remove_all_containers +kill_all_container_shims diff --git a/microk8s-resources/wrappers/apiservice-kicker b/microk8s-resources/wrappers/apiservice-kicker index a29c38835a..fb353d51c3 100755 --- a/microk8s-resources/wrappers/apiservice-kicker +++ b/microk8s-resources/wrappers/apiservice-kicker @@ -65,15 +65,9 @@ do echo "cert change detected. Reconfiguring the kube-apiserver" rm -rf .srl snapctl stop microk8s.daemon-kubelite - if is_strict - then - remove_all_containers - snapctl restart microk8s.daemon-containerd - else - snapctl stop microk8s.daemon-containerd - kill_all_container_shims - snapctl start microk8s.daemon-containerd - fi + remove_all_containers + kill_all_container_shims + snapctl restart microk8s.daemon-containerd snapctl start microk8s.daemon-kubelite start_all_containers restart_attempt=$[$restart_attempt+1] diff --git a/microk8s-resources/wrappers/microk8s-stop.wrapper b/microk8s-resources/wrappers/microk8s-stop.wrapper index c942d59e92..4de540e513 100755 --- a/microk8s-resources/wrappers/microk8s-stop.wrapper +++ b/microk8s-resources/wrappers/microk8s-stop.wrapper @@ -40,13 +40,13 @@ while true; do esac done -stopcmd="run_with_sudo snap stop ${SNAP_NAME} --disable" +prefix_cmd="run_with_sudo snap" if is_strict then - stopcmd="snapctl stop microk8s.daemon-kubelite --disable" + prefix_cmd="snapctl" fi -$stopcmd +$prefix_cmd stop microk8s.daemon-kubelite --disable stop_status=$? if ! [ $stop_status -eq 0 ] @@ -54,12 +54,8 @@ then echo 'Failed to stop microk8s services. Check snapd logs with "journalctl -u snapd.service"' exit 1 else - if is_strict - then - remove_all_containers - snapctl stop microk8s --disable - else - kill_all_container_shims - fi + remove_all_containers + kill_all_container_shims + $prefix_cmd stop microk8s --disable run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock fi diff --git a/snap/hooks/remove b/snap/hooks/remove index f4e8932280..e0aa5d4c97 100755 --- a/snap/hooks/remove +++ b/snap/hooks/remove @@ -7,6 +7,15 @@ use_snap_env snapctl stop ${SNAP_NAME}.daemon-kubelite 2>&1 || true +# Temporarily start containerd so we can stop and kill all the microk8s containers. +snapctl start ${SNAP_NAME}.daemon-containerd 2>&1 || true +# wait for containerd to start. +sleep 5 + +# Remove any lingering containers and shims. +remove_all_containers +kill_all_container_shims + # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box. if test -L /var/lib/kubelet; then unlink /var/lib/kubelet || true diff --git a/tests/requirements.txt b/tests/requirements.txt index 93f8fdc657..dbe0f5b667 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,5 +7,6 @@ pyyaml sh jsonschema==4.0.0 pdbpp +psutil netifaces requests diff --git a/tests/test-simple.py b/tests/test-simple.py index deeb471778..757b9f1ef2 100644 --- a/tests/test-simple.py +++ b/tests/test-simple.py @@ -3,6 +3,8 @@ import requests import os.path +import utils + class TestSimple(object): def test_microk8s_nodes_ready(self): @@ -132,3 +134,17 @@ def test_microk8s_services_running(self): # Verify that all node services are running assert running_node_services == set(node_services), "Not all node services are running" + + def test_microk8s_stop_start(self): + coredns_procs = utils._get_process("coredns") + assert len(coredns_procs) > 0, "Expected to find a coredns process running." + + utils.run_until_success("/snap/bin/microk8s.stop", timeout_insec=180) + + new_coredns_procs = utils._get_process("coredns") + assert len(new_coredns_procs) == 0, "coredns found still running after microk8s stop." + + utils.run_until_success("/snap/bin/microk8s.start", timeout_insec=180) + + new_coredns_procs = utils._get_process("coredns") + assert len(new_coredns_procs) > 0, "Expected to find a new coredns process running." diff --git a/tests/test-upgrade.py b/tests/test-upgrade.py index 68dc7b0d05..279688f358 100644 --- a/tests/test-upgrade.py +++ b/tests/test-upgrade.py @@ -20,6 +20,7 @@ is_container, is_ipv6_configured, kubectl, + _get_process, ) upgrade_from = os.environ.get("UPGRADE_MICROK8S_FROM", "beta") @@ -157,3 +158,5 @@ def test_upgrade(self): if not is_container(): # On lxc umount docker overlay is not permitted. check_call("sudo snap remove microk8s".split()) + coredns_procs = _get_process("coredns") + assert len(coredns_procs) == 0, "Expected to have 0 coredns processes running." diff --git a/tests/utils.py b/tests/utils.py index 0dba44002f..b81c836d08 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,6 +3,7 @@ import time import yaml import platform +import psutil from subprocess import check_output, CalledProcessError, check_call @@ -270,3 +271,7 @@ def is_ipv6_configured(): return b"inet6" in output except CalledProcessError: return False + + +def _get_process(name): + return [p for p in psutil.process_iter() if name == p.name()]