feat(node drain): add DetachManuallyAttachedVolumesWhenCordoned setting

ref: longhorn/longhorn 6978

Signed-off-by: Jack Lin <[email protected]>

controller/volume_attachment_controller.go

@@ -20,6 +20,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 
+	"github.com/longhorn/longhorn-manager/constant"
 	"github.com/longhorn/longhorn-manager/datastore"
 	"github.com/longhorn/longhorn-manager/types"
 
@@ -91,6 +92,11 @@ func NewLonghornVolumeAttachmentController(
 	}, 0)
 	vac.cacheSyncs = append(vac.cacheSyncs, ds.EngineInformer.HasSynced)
 
+	ds.KubeNodeInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
+		UpdateFunc: func(old, cur interface{}) { vac.enqueueNodeChange(cur) },
+	}, 0)
+	vac.cacheSyncs = append(vac.cacheSyncs, ds.KubeNodeInformer.HasSynced)
+
 	return vac
 }
 
@@ -169,6 +175,34 @@ func (vac *VolumeAttachmentController) enqueueEngineChange(obj interface{}) {
 
 }
 
+func (vac *VolumeAttachmentController) enqueueNodeChange(obj interface{}) {
+	kubernetesNode, ok := obj.(*corev1.Node)
+	if !ok {
+		deletedState, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			utilruntime.HandleError(fmt.Errorf("received unexpected obj: %#v", obj))
+			return
+		}
+
+		// use the last known state, to enqueue, dependent objects
+		kubernetesNode, ok = deletedState.Obj.(*corev1.Node)
+		if !ok {
+			utilruntime.HandleError(fmt.Errorf("DeletedFinalStateUnknown contained invalid object: %#v", deletedState.Obj))
+			return
+		}
+	}
+
+	volumeAttachments, err := vac.ds.ListLHVolumeAttachmentsRO()
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failed to list VolumeAttachments when enqueuing node %v: %v", kubernetesNode.Name, err))
+		return
+	}
+
+	for _, va := range volumeAttachments {
+		vac.enqueueVolumeAttachment(va)
+	}
+}
+
 func (vac *VolumeAttachmentController) Run(workers int, stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 	defer vac.queue.ShutDown()
@@ -269,6 +303,11 @@ func (vac *VolumeAttachmentController) reconcile(vaName string) (err error) {
 			}
 
 		}
+		if !reflect.DeepEqual(existingVA.Spec, va.Spec) {
+			if _, err = vac.ds.UpdateLHVolumeAttachment(va); err != nil {
+				return
+			}
+		}
 		if !reflect.DeepEqual(existingVA.Status, va.Status) {
 			if _, err = vac.ds.UpdateLHVolumeAttachmentStatus(va); err != nil {
 				return
@@ -280,6 +319,8 @@ func (vac *VolumeAttachmentController) reconcile(vaName string) (err error) {
 	// Note that in this controller the desire state is recorded in VA.Spec
 	// and the current state of the world is recorded inside volume CR
 
+	vac.handleNodeCordoned(va, vol)
+
 	vac.handleVolumeDetachment(va, vol)
 
 	vac.handleVolumeAttachment(va, vol)
@@ -289,6 +330,43 @@ func (vac *VolumeAttachmentController) reconcile(vaName string) (err error) {
 	return vac.handleVAStatusUpdate(va, vol)
 }
 
+// handleNodeCordoned delete ui attachment ticket from the va when the target node is cordened
+func (vac *VolumeAttachmentController) handleNodeCordoned(va *longhorn.VolumeAttachment, vol *longhorn.Volume) {
+	log := getLoggerForLHVolumeAttachment(vac.logger, va)
+
+	detachManuallyAttachedVolumesWhenCordoned, err := vac.ds.GetSettingAsBool(types.SettingNameDetachManuallyAttachedVolumesWhenCordoned)
+	if err != nil {
+		log.WithError(err).Warnf("Failed to get setting %v", types.SettingNameDetachManuallyAttachedVolumesWhenCordoned)
+		return
+	}
+
+	var manualAttachmentTicket *longhorn.AttachmentTicket
+	for _, attachmentTicket := range va.Spec.AttachmentTickets {
+		if attachmentTicket.Type == longhorn.AttacherTypeLonghornAPI {
+			manualAttachmentTicket = attachmentTicket
+			break
+		}
+	}
+
+	if manualAttachmentTicket != nil {
+		unschedulable, err := vac.ds.IsKubeNodeUnschedulable(manualAttachmentTicket.NodeID)
+		if err != nil {
+			log.WithError(err).Warnf("Failed to check if node %v schedulable", manualAttachmentTicket.NodeID)
+			return
+		}
+		if unschedulable {
+			if !detachManuallyAttachedVolumesWhenCordoned {
+				msg := fmt.Sprintf("Failed to detach manually attached volume due to %v is set to false", types.SettingNameDetachManuallyAttachedVolumesWhenCordoned)
+				vac.eventRecorder.Event(va, corev1.EventTypeWarning, constant.EventReasonDetachedUnexpectedly, msg)
+				return
+			}
+
+			log.Infof("Deleting manual attachment ticket %v due to node is unschedulable", manualAttachmentTicket.ID)
+			delete(va.Spec.AttachmentTickets, manualAttachmentTicket.ID)
+		}
+	}
+}
+
 func (vac *VolumeAttachmentController) handleVolumeMigration(va *longhorn.VolumeAttachment, vol *longhorn.Volume) {
 	if !isMigratableVolume(vol) {
 		return

datastore/longhorn.go

@@ -4738,6 +4738,11 @@ func (s *DataStore) ListLonghornVolumeAttachmentByVolumeRO(name string) ([]*long
 	return s.lhVolumeAttachmentLister.VolumeAttachments(s.namespace).List(volumeSelector)
 }
 
+// ListLHVolumeAttachmentsRO returns a list of all VolumeAttachments for the given namespace
+func (s *DataStore) ListLHVolumeAttachmentsRO() ([]*longhorn.VolumeAttachment, error) {
+	return s.lhVolumeAttachmentLister.VolumeAttachments(s.namespace).List(labels.Everything())
+}
+
 // RemoveFinalizerForLHVolumeAttachment will result in deletion if DeletionTimestamp was set
 func (s *DataStore) RemoveFinalizerForLHVolumeAttachment(va *longhorn.VolumeAttachment) error {
 	if !util.FinalizerExists(longhornFinalizerKey, va) {

types/setting.go

@@ -74,6 +74,7 @@ const (
 	SettingNameReplicaZoneSoftAntiAffinity                              = SettingName("replica-zone-soft-anti-affinity")
 	SettingNameNodeDownPodDeletionPolicy                                = SettingName("node-down-pod-deletion-policy")
 	SettingNameNodeDrainPolicy                                          = SettingName("node-drain-policy")
+	SettingNameDetachManuallyAttachedVolumesWhenCordoned                = SettingName("detach-manually-attached-volumes-when-cordoned")
 	SettingNamePriorityClass                                            = SettingName("priority-class")
 	SettingNameDisableRevisionCounter                                   = SettingName("disable-revision-counter")
 	SettingNameReplicaReplenishmentWaitInterval                         = SettingName("replica-replenishment-wait-interval")
@@ -151,6 +152,7 @@ var (
 		SettingNameReplicaZoneSoftAntiAffinity,
 		SettingNameNodeDownPodDeletionPolicy,
 		SettingNameNodeDrainPolicy,
+		SettingNameDetachManuallyAttachedVolumesWhenCordoned,
 		SettingNamePriorityClass,
 		SettingNameDisableRevisionCounter,
 		SettingNameReplicaReplenishmentWaitInterval,
@@ -254,6 +256,7 @@ var (
 		SettingNameReplicaZoneSoftAntiAffinity:                              SettingDefinitionReplicaZoneSoftAntiAffinity,
 		SettingNameNodeDownPodDeletionPolicy:                                SettingDefinitionNodeDownPodDeletionPolicy,
 		SettingNameNodeDrainPolicy:                                          SettingDefinitionNodeDrainPolicy,
+		SettingNameDetachManuallyAttachedVolumesWhenCordoned:                SettingDefinitionDetachManuallyAttachedVolumesWhenCordoned,
 		SettingNamePriorityClass:                                            SettingDefinitionPriorityClass,
 		SettingNameDisableRevisionCounter:                                   SettingDefinitionDisableRevisionCounter,
 		SettingNameReplicaReplenishmentWaitInterval:                         SettingDefinitionReplicaReplenishmentWaitInterval,
@@ -719,6 +722,16 @@ var (
 		},
 	}
 
+	SettingDefinitionDetachManuallyAttachedVolumesWhenCordoned = SettingDefinition{
+		DisplayName: "Detach Manually Attached Volumes When Cordoned",
+		Description: "Automatically detach volumes that are attached manually when node is cordoned.",
+		Category:    SettingCategoryGeneral,
+		Type:        SettingTypeBool,
+		Required:    true,
+		ReadOnly:    false,
+		Default:     "false",
+	}
+
 	SettingDefinitionPriorityClass = SettingDefinition{
 		DisplayName: "Priority Class",
 		Description: "The name of the Priority Class to set on the Longhorn components. This can help prevent Longhorn components from being evicted under Node Pressure. \n" +
@@ -1294,6 +1307,8 @@ func ValidateSetting(name, value string) (err error) {
 		fallthrough
 	case SettingNameAllowCollectingLonghornUsage:
 		fallthrough
+	case SettingNameDetachManuallyAttachedVolumesWhenCordoned:
+		fallthrough
 	case SettingNameReplicaDiskSoftAntiAffinity:
 		fallthrough
 	case SettingNameDisableSnapshotPurge: