Skip to content

Commit

Permalink
Fixing a bug in updating status after on-demand operation. (#301)
Browse files Browse the repository at this point in the history
  • Loading branch information
tanmayja authored Jul 12, 2024
1 parent efa1ba4 commit 13514b7
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 8 deletions.
8 changes: 4 additions & 4 deletions controllers/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -1537,13 +1537,13 @@ func (r *SingleClusterReconciler) updateOperationStatus(restartedASDPodNames, re
statusPods := sets.New(statusOp.PodList...)

if statusOp.Kind == asdbv1.OperationWarmRestart {
if quickRestartsSet != nil {
if quickRestartsSet.Len() > 0 {
statusOp.PodList = statusPods.Union(quickRestartsSet.Intersection(specPods)).UnsortedList()
}

// If the operation is a warm restart and the pod undergoes a cold restart for any reason,
// we will still consider the warm restart operation as completed for that pod.
if podRestartsSet != nil {
if podRestartsSet.Len() > 0 {
statusOp.PodList = statusPods.Union(podRestartsSet.Intersection(specPods)).UnsortedList()
}
}
Expand All @@ -1561,13 +1561,13 @@ func (r *SingleClusterReconciler) updateOperationStatus(restartedASDPodNames, re
var podList []string

if specOp.Kind == asdbv1.OperationWarmRestart {
if quickRestartsSet != nil {
if quickRestartsSet.Len() > 0 {
podList = quickRestartsSet.Intersection(specPods).UnsortedList()
}

// If the operation is a warm restart and the pod undergoes a cold restart for any reason,
// we will still consider the warm restart operation as completed for that pod.
if podRestartsSet != nil {
if podRestartsSet.Len() > 0 {
podList = append(podList, podRestartsSet.Intersection(specPods).UnsortedList()...)
}
}
Expand Down
8 changes: 4 additions & 4 deletions controllers/rack.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,15 @@ func (r *SingleClusterReconciler) reconcileRacks() reconcileResult {
// remove ignorable pods from failedPods
failedPods = getNonIgnorablePods(failedPods, ignorablePodNames)
if len(failedPods) != 0 {
r.Log.Info("Reconcile the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", failedPods)
r.Log.Info("Reconcile the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", getPodNames(failedPods))

if res = r.reconcileRack(
found, state, ignorablePodNames, failedPods,
); !res.isSuccess {
return res
}

r.Log.Info("Reconciled the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", failedPods)
r.Log.Info("Reconciled the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", getPodNames(failedPods))
}

// 2. Again, fetch the pods for the rack and if there are failed pods then restart them.
Expand All @@ -114,14 +114,14 @@ func (r *SingleClusterReconciler) reconcileRacks() reconcileResult {
// remove ignorable pods from failedPods
failedPods = getNonIgnorablePods(failedPods, ignorablePodNames)
if len(failedPods) != 0 {
r.Log.Info("Restart the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", failedPods)
r.Log.Info("Restart the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", getPodNames(failedPods))

if _, res = r.rollingRestartRack(found, state, ignorablePodNames, nil,
failedPods); !res.isSuccess {
return res
}

r.Log.Info("Restarted the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", failedPods)
r.Log.Info("Restarted the failed pods in the Rack", "rackID", state.Rack.ID, "failedPods", getPodNames(failedPods))
// Requeue after 1 second to fetch latest CR object with updated pod status
return reconcileRequeueAfter(1)
}
Expand Down
1 change: 1 addition & 0 deletions test/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ func clusterWithMaxIgnorablePod(ctx goctx.Context) {
err = k8sClient.Update(ctx, pod)
Expect(err).ToNot(HaveOccurred())

// Underlying kubernetes cluster should have atleast 6 nodes to run this test successfully.
By("Delete rack with id 2")
aeroCluster, err = getCluster(k8sClient, ctx, clusterNamespacedName)
Expect(err).ToNot(HaveOccurred())
Expand Down

0 comments on commit 13514b7

Please sign in to comment.