Cleans up containers and shims on microk8s stop and snap removal (#4755)

The containerd shims and the containers are now removed in both classic
and strict modes when stopping microk8s (microk8s stop) or when removing
the snap.

Note that ctr container delete has no ``--force`` flag.

Adds additional assertion in the upgrade test, ensuring that the Pods /
containers have been removed on snap removal.

Adds additional test, ensuring that the Pods / containers have been
removed when running microk8s stop.

.github/workflows/build-snap.yml

@@ -48,7 +48,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap
@@ -73,7 +73,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap
@@ -136,7 +136,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap

microk8s-resources/actions/common/utils.sh

@@ -988,7 +988,7 @@ remove_all_containers() {
 
     for container in $("${SNAP}/microk8s-ctr.wrapper" containers ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}')
     do
-        "${SNAP}/microk8s-ctr.wrapper" container delete --force $container &>/dev/null || true
+        "${SNAP}/microk8s-ctr.wrapper" container delete $container &>/dev/null || true
     done
 }
 

microk8s-resources/default-hooks/remove.d/90-containers

@@ -2,9 +2,5 @@
 
 . "${SNAP}/actions/common/utils.sh"
 
-if is_strict
-then
-  remove_all_containers
-else
-  kill_all_container_shims
-fi
+remove_all_containers
+kill_all_container_shims

microk8s-resources/wrappers/apiservice-kicker

@@ -65,15 +65,9 @@ do
         echo "cert change detected. Reconfiguring the kube-apiserver"
         rm -rf .srl
         snapctl stop microk8s.daemon-kubelite
-        if is_strict
-        then
-          remove_all_containers
-          snapctl restart microk8s.daemon-containerd
-        else
-          snapctl stop microk8s.daemon-containerd
-          kill_all_container_shims
-          snapctl start microk8s.daemon-containerd
-        fi
+        remove_all_containers
+        kill_all_container_shims
+        snapctl restart microk8s.daemon-containerd
         snapctl start microk8s.daemon-kubelite
         start_all_containers
         restart_attempt=$[$restart_attempt+1]

microk8s-resources/wrappers/microk8s-stop.wrapper

@@ -40,26 +40,22 @@ while true; do
     esac
 done
 
-stopcmd="run_with_sudo snap stop ${SNAP_NAME} --disable"
+prefix_cmd="run_with_sudo snap"
 if is_strict
 then
-  stopcmd="snapctl stop microk8s.daemon-kubelite --disable"
+  prefix_cmd="snapctl"
 fi
 
-$stopcmd
+$prefix_cmd stop microk8s.daemon-kubelite --disable
 stop_status=$?
 
 if ! [ $stop_status -eq 0 ]
 then
     echo 'Failed to stop microk8s services. Check snapd logs with "journalctl -u snapd.service"'
     exit 1
 else
-    if is_strict
-    then
-      remove_all_containers
-      snapctl stop microk8s --disable
-    else
-      kill_all_container_shims
-    fi
+    remove_all_containers
+    kill_all_container_shims
+    $prefix_cmd stop microk8s --disable
     run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock
 fi

snap/hooks/remove

@@ -7,6 +7,15 @@ use_snap_env
 
 snapctl stop ${SNAP_NAME}.daemon-kubelite 2>&1 || true
 
+# Temporarily start containerd so we can stop and kill all the microk8s containers.
+snapctl start ${SNAP_NAME}.daemon-containerd 2>&1 || true
+# wait for containerd to start.
+sleep 5
+
+# Remove any lingering containers and shims.
+remove_all_containers
+kill_all_container_shims
+
 # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box.
 if test -L /var/lib/kubelet; then
   unlink /var/lib/kubelet || true

tests/requirements.txt

@@ -7,5 +7,6 @@ pyyaml
 sh
 jsonschema==4.0.0
 pdbpp
+psutil
 netifaces
 requests

tests/test-simple.py

@@ -3,6 +3,8 @@
 import requests
 import os.path
 
+import utils
+
 
 class TestSimple(object):
     def test_microk8s_nodes_ready(self):
@@ -132,3 +134,17 @@ def test_microk8s_services_running(self):
 
         # Verify that all node services are running
         assert running_node_services == set(node_services), "Not all node services are running"
+
+    def test_microk8s_stop_start(self):
+        coredns_procs = utils._get_process("coredns")
+        assert len(coredns_procs) > 0, "Expected to find a coredns process running."
+
+        utils.run_until_success("/snap/bin/microk8s.stop", timeout_insec=180)
+
+        new_coredns_procs = utils._get_process("coredns")
+        assert len(new_coredns_procs) == 0, "coredns found still running after microk8s stop."
+
+        utils.run_until_success("/snap/bin/microk8s.start", timeout_insec=180)
+
+        new_coredns_procs = utils._get_process("coredns")
+        assert len(new_coredns_procs) > 0, "Expected to find a new coredns process running."

tests/test-upgrade.py

@@ -20,6 +20,7 @@
     is_container,
     is_ipv6_configured,
     kubectl,
+    _get_process,
 )
 
 upgrade_from = os.environ.get("UPGRADE_MICROK8S_FROM", "beta")
@@ -157,3 +158,5 @@ def test_upgrade(self):
         if not is_container():
             # On lxc umount docker overlay is not permitted.
             check_call("sudo snap remove microk8s".split())
+            coredns_procs = _get_process("coredns")
+            assert len(coredns_procs) == 0, "Expected to have 0 coredns processes running."

tests/utils.py

@@ -3,6 +3,7 @@
 import time
 import yaml
 import platform
+import psutil
 from subprocess import check_output, CalledProcessError, check_call
 
 
@@ -270,3 +271,7 @@ def is_ipv6_configured():
         return b"inet6" in output
     except CalledProcessError:
         return False
+
+
+def _get_process(name):
+    return [p for p in psutil.process_iter() if name == p.name()]