skip completed AWs after MCAD restart

pkg/controller/queuejob/queuejob_controller_ex.go

@@ -640,7 +640,7 @@ func (qjm *XController) GetAggregatedResources(cqj *arbv1.AppWrapper) *clusterst
 		}
 		allocated = allocated.Add(qjv)
 	}
-
+	klog.V(2).Infof("[GetAggregatedResources] Total resources needed to dispatch AW as reported in custompodresources is %v", allocated)
 	return allocated
 }
 
@@ -1626,7 +1626,10 @@ func (cc *XController) addQueueJob(obj interface{}) {
 					if latestAw.Status.State != arbv1.AppWrapperStateActive && latestAw.Status.State != arbv1.AppWrapperStateEnqueued && latestAw.Status.State != arbv1.AppWrapperStateRunningHoldCompletion {
 						klog.V(2).Infof("[Informer-addQJ] Stopping requeue for AW %s/%s with status %s", latestAw.Namespace, latestAw.Name, latestAw.Status.State)
 						AwinEtcd, err := cc.arbclients.WorkloadV1beta1().AppWrappers(latestAw.Namespace).Get(context.Background(), latestAw.Name, metav1.GetOptions{})
-						if AwinEtcd.Status.State == latestAw.Status.State && err != nil {
+						if apierrors.IsNotFound(err) {
+							//AW is already completed when MCAD is restarted.
+							break
+						} else if AwinEtcd.Status.State == latestAw.Status.State && err != nil {
 							break // Exit the loop
 						}
 					}