only retry rate limited nodes

src/actions.py

@@ -353,8 +353,7 @@ def queryNodes(self, indexes=None, apimodule=False, options=False):
             #Init status messages
             statuscount = {}
             errorcount = 0
-            laststatus = None
-            laststatuscount = 0
+            ratelimitcount = 0
             allowedstatus = ['fetched (200)','downloaded (200)','fetched (202)','stream'] #,'error (400)'
 
 
@@ -427,69 +426,75 @@ def queryNodes(self, indexes=None, apimodule=False, options=False):
                             if not job['nodeindex'].isValid():
                                 continue
 
-                            #add data
+                            # Add data
                             treeindex = job['nodeindex']
                             treenode = treeindex.internalPointer()
                             treenode.appendNodes(job['data'], job['options'], job['headers'], True)
                             if options.get('expand',False):
                                  self.mainWindow.tree.setExpanded(treeindex,True)
 
-                            # Show status
+                            # Count status
                             status = job['options'].get('querystatus', 'empty')
-                            ratelimit = job['options'].get('ratelimit', False)
-                            count = 1 if not status in statuscount else statuscount[status]+1
-                            statuscount[status] = count
-                            progress.showInfo(status,u"{} response(s) with status: {}".format(count,status))
-                            progress.showInfo('newnodes',u"{} new node(s) created".format(self.mainWindow.tree.treemodel.nodecounter))
-                            progress.showInfo('threads',u"{} active thread(s)".format(threadpool.getThreadCount()))
-                            progress.setRemaining(threadpool.getJobCount())
-
-                            # Custom info from modules
-                            info = job['options'].get('info', {})
-                            for name, value in info.iteritems():
-                                progress.showInfo(name, value)
 
-                            # Count status
-                            if (status != laststatus):
-                                laststatus=status
-                                laststatuscount = 1
+                            if not status in statuscount:
+                                statuscount[status] = 1
                             else:
-                                laststatuscount += 1
+                                statuscount[status] = statuscount[status]+1
 
                             # Collect errors for automatic retry
-                            if not (status in allowedstatus):
+                            if not status in allowedstatus:
                                 threadpool.addError(job)
+                                errorcount += 1
+
+                            # Detect rate limit
+                            ratelimit = job['options'].get('ratelimit', False)
+
+                            if ratelimit:
+                                ratelimitcount += 1
 
                             # Clear errors
-                            if (laststatus in allowedstatus) and (laststatuscount == 1) and not ratelimit:
+                            if not threadpool.suspended and (status in allowedstatus) and not ratelimit:
                                 threadpool.clearRetry()
+                                errorcount = 0
+                                ratelimitcount = 0
+
 
                             # Suspend on error
-                            elif not (laststatus in allowedstatus) and ((laststatuscount > (globaloptions['errors']-1)) or ratelimit):
+                            elif (errorcount > (globaloptions['errors']-1)) or (ratelimitcount > 0):
                                 threadpool.suspendJobs()
 
                                 if ratelimit:
                                     msg = "You reached the rate limit of the API."
                                 else:
-                                    msg = "{} consecutive errors occurred.\nPlease check your settings.".format(laststatuscount)
+                                    msg = "{} consecutive errors occurred.\nPlease check your settings.".format(errorcount)
 
                                 timeout = 60 * 5 #5 minutes
 
                                 # Adjust progress
                                 progress.setRemaining(threadpool.getJobCount() + threadpool.getRetryCount())
-                                progress.showError(msg, timeout)
+                                progress.showError(msg, timeout, ratelimitcount > 0)
                                 self.mainWindow.tree.treemodel.commitNewNodes()
 
+                            # Show info
+                            progress.showInfo(status,u"{} response(s) with status: {}".format(statuscount[status],status))
+                            progress.showInfo('newnodes',u"{} new node(s) created".format(self.mainWindow.tree.treemodel.nodecounter))
+                            progress.showInfo('threads',u"{} active thread(s)".format(threadpool.getThreadCount()))
+                            progress.setRemaining(threadpool.getJobCount())
+
+                            # Custom info from modules
+                            info = job['options'].get('info', {})
+                            for name, value in info.iteritems():
+                                progress.showInfo(name, value)
+
                         # Abort
                         elif progress.wasCanceled:
                             progress.showInfo('cancel', u"Disconnecting from stream, may take some time.")
                             threadpool.stopJobs()
 
                         # Retry
                         elif progress.wasResumed:
-                            laststatuscount = 1
-                            if progress.retry:
-                                threadpool.retryJobs(ratelimitonly=True)
+                            if progress.wasRetried:
+                                threadpool.retryJobs()
                             else:
                                 threadpool.clearRetry()
                                 threadpool.resumeJobs()

src/apithread.py

@@ -104,10 +104,10 @@ def addError(self, job):
                   'options': deepcopy(job['options'])}
         self.errors.put(newjob)
 
-    def retryJobs(self, ratelimitonly = False):
+    def retryJobs(self):
         while not self.errors.empty():
             newjob = self.errors.get()
-            if not ratelimitonly or newjob['options'].get('ratelimit', False):
+            if newjob['options'].get('ratelimit', False):
                 newjob['number'] = self.jobcount
                 self.jobcount += 1
                 self.input.appendleft(newjob)

src/progressbar.py

@@ -13,12 +13,13 @@ def __init__(self, mainmessage, parent=None):
         self.timeout = 60
         self.timer = None
         self.countdown = False
+        self.retryoption = False
 
         self.rate_update_frequency = 3
         self.rate_interval = 30
         self.rate_values = []
 
-        self.retry = False
+        self.wasRetried = False
         self.wasResumed = False
 
         #Create layout
@@ -64,18 +65,21 @@ def __init__(self, mainmessage, parent=None):
         self.open()
         self.hideError()
 
-    def showError(self, msg, timeout = 60):
+    def showError(self, msg, timeout = 60, retryoption = False):
         if self.countdown == False:
             self.countdown = True
             self.wasResumed = False
-            self.retry = False
+            self.wasRetried = False
+            self.retryoption = retryoption
             self.timeout = timeout
 
             self.errorLabel.setText(msg)
             self.errorLabel.show()
 
-            self.retryButton.setText(u"Retry")
-            self.retryButton.show()
+            if self.retryoption:
+                self.retryButton.setText(u"Retry")
+                self.retryButton.show()
+
             self.skipButton.setText(u"Skip")
             self.skipButton.show()
 
@@ -85,7 +89,7 @@ def hideError(self):
         self.countdown = False
 
         self.wasResumed = False
-        self.retry = False
+        self.wasRetried = False
 
         self.errorLabel.hide()
         self.retryButton.hide()
@@ -95,10 +99,13 @@ def timerEvent(self):
         self.timeout -= 1
         if (self.timeout <= 0):
             self.doretry()
-        else:
+        elif self.retryoption:
             self.retryButton.setText(u"Retry [{}]".format(self.timeout))
             self.skipButton.setText(u"Skip")
-
+            self.startCountdown()
+        else:
+            self.skipButton.setText(u"Skip [{}]".format(self.timeout))
+            self.retryButton.setText(u"Retry")
             self.startCountdown()
 
     def startCountdown(self):
@@ -119,15 +126,15 @@ def doskip(self):
         self.stopCountdown()
         self.countdown = False
         self.timeout = 0
-        self.retry = False
+        self.wasRetried = False
         self.wasResumed = True
 
 
     def doretry(self):
         self.stopCountdown()
         self.countdown = False
         self.timeout = 0
-        self.retry = True
+        self.wasRetried = True
         self.wasResumed = True