Rate limit requeuing node for eviction

Longhorn 2238 Signed-off-by: Eric Weber <[email protected]>
ejweber · Nov 22, 2023 · 8e2a3bd · 8e2a3bd
1 parent d2583a3
commit 8e2a3bd
Showing 1 changed file with 12 additions and 2 deletions.
diff --git a/controller/node_controller.go b/controller/node_controller.go
@@ -570,6 +570,16 @@ func (nc *NodeController) enqueueNode(obj interface{}) {
 	nc.queue.Add(key)
 }
 
+func (nc *NodeController) enqueueNodeRateLimited(obj interface{}) {
+	key, err := controller.KeyFunc(obj)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", obj, err))
+		return
+	}
+
+	nc.queue.AddRateLimited(key)
+}
+
 func (nc *NodeController) enqueueSetting(obj interface{}) {
 	nodes, err := nc.ds.ListNodesRO()
 	if err != nil {
@@ -1491,15 +1501,15 @@ func (nc *NodeController) syncReplicaEvictionRequested(node *longhorn.Node, kube
 			replicaLog.Infof("Requesting replica eviction")
 			if _, err := nc.ds.UpdateReplica(replicaToSync.Replica); err != nil {
 				replicaLog.Warn("Failed to request replica eviction, will enqueue then resync node")
-				nc.enqueueNode(node)
+				nc.enqueueNodeRateLimited(node)
 				continue
 			}
 			nc.eventRecorder.Eventf(replicaToSync.Replica, corev1.EventTypeNormal, replicaToSync.syncReason, "Requesting replica %v eviction from node %v and disk %v", replicaToSync.Name, node.Spec.Name, replicaToSync.Spec.DiskID)
 		} else {
 			replicaLog.Infof("Cancelling replica eviction")
 			if _, err := nc.ds.UpdateReplica(replicaToSync.Replica); err != nil {
 				replicaLog.Warn("Failed to cancel replica eviction, will enqueue then resync node")
-				nc.enqueueNode(node)
+				nc.enqueueNodeRateLimited(node)
 				continue
 			}
 			nc.eventRecorder.Eventf(replicaToSync.Replica, corev1.EventTypeNormal, replicaToSync.syncReason, "Cancelling replica %v eviction from node %v and disk %v", replicaToSync.Name, node.Spec.Name, replicaToSync.Spec.DiskID)