Waiting for old Longhorn manager to be fully removed before starting

upgrade path

Fix the issue that both old and new Longhorn managers are running
at the same time causing the problem for the upgrade logic

longhorn-6294

Signed-off-by: Phan Le <[email protected]>

app/daemon.go

@@ -149,7 +149,7 @@ func startManager(c *cli.Context) error {
 		return err
 	}
 
-	if err := upgrade.Upgrade(kubeconfigPath, currentNodeID); err != nil {
+	if err := upgrade.Upgrade(kubeconfigPath, currentNodeID, managerImage); err != nil {
 		return err
 	}
 

datastore/kubernetes.go

@@ -51,11 +51,7 @@ func labelMapToLabelSelector(labels map[string]string) (labels.Selector, error)
 }
 
 func (s *DataStore) GetManagerLabel() map[string]string {
-	return map[string]string{
-		// TODO standardize key
-		// longhornSystemKey: longhornSystemManager,
-		"app": types.LonghornManagerDaemonSetName,
-	}
+	return types.GetManagerLabels()
 }
 
 func (s *DataStore) getManagerSelector() (labels.Selector, error) {

types/types.go

@@ -388,6 +388,11 @@ func GetLonghornLabelCRDAPIVersionKey() string {
 	return GetLonghornLabelKey(LonghornLabelCRDAPIVersion)
 }
 
+func GetManagerLabels() map[string]string {
+	return map[string]string{
+		"app": LonghornManagerDaemonSetName,
+	}
+}
 func GetEngineImageLabels(engineImageName string) map[string]string {
 	labels := GetBaseLabelsForSystemManagedComponent()
 	labels[GetLonghornLabelComponentKey()] = LonghornLabelEngineImage

upgrade/upgrade.go

@@ -34,7 +34,7 @@ const (
 	LeaseLockName = "longhorn-manager-upgrade-lock"
 )
 
-func Upgrade(kubeconfigPath, currentNodeID string) error {
+func Upgrade(kubeconfigPath, currentNodeID, managerImage string) error {
 	namespace := os.Getenv(types.EnvPodNamespace)
 	if namespace == "" {
 		logrus.Warnf("Cannot detect pod namespace, environment variable %v is missing, "+
@@ -70,6 +70,10 @@ func Upgrade(kubeconfigPath, currentNodeID string) error {
 		return err
 	}
 
+	if err := waitForOldLonghornManagersToBeFullyRemoved(namespace, managerImage, kubeClient); err != nil {
+		return err
+	}
+
 	if err := upgrade(currentNodeID, namespace, config, lhClient, kubeClient); err != nil {
 		return err
 	}
@@ -249,3 +253,38 @@ func doResourceUpgrade(namespace string, lhClient *lhclientset.Clientset, kubeCl
 
 	return upgradeutil.CreateOrUpdateLonghornVersionSetting(namespace, lhClient)
 }
+
+func waitForOldLonghornManagersToBeFullyRemoved(namespace, managerImage string, kubeClient *clientset.Clientset) error {
+	logrus.Info("Waiting for old Longhorn manager pods to be fully removed")
+	for i := 0; i < 600; i++ {
+		managerPods, err := upgradeutil.ListManagerPods(namespace, kubeClient)
+		if err != nil {
+			return err
+		}
+		foundOldManager := false
+		for _, pod := range managerPods {
+			if isOldPod, oldImage := isOldManagerPod(pod, managerImage); isOldPod {
+				logrus.Infof("Found old longhorn manager: %v with image %v", pod.Name, oldImage)
+				foundOldManager = true
+				break
+			}
+		}
+		if !foundOldManager {
+			return nil
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	return fmt.Errorf("timed out while waiting for old Longhorn manager pods to be fully removed")
+}
+
+func isOldManagerPod(pod corev1.Pod, managerImage string) (bool, string) {
+	for _, container := range pod.Spec.Containers {
+		if container.Name == "longhorn-manager" {
+			if container.Image != managerImage {
+				return true, container.Image
+			}
+		}
+	}
+	return false, ""
+}

upgrade/util/util.go

@@ -110,6 +110,16 @@ func ListIMPods(namespace string, kubeClient *clientset.Clientset) ([]v1.Pod, er
 	return imPodsList.Items, nil
 }
 
+func ListManagerPods(namespace string, kubeClient *clientset.Clientset) ([]v1.Pod, error) {
+	managerPodsList, err := kubeClient.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{
+		LabelSelector: labels.Set(types.GetManagerLabels()).String(),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return managerPodsList.Items, nil
+}
+
 func MergeStringMaps(baseMap, overwriteMap map[string]string) map[string]string {
 	result := map[string]string{}
 	for k, v := range baseMap {