Merge pull request #55 from nlnwa/fix-fetch-timeout-worker-loop

Fix potential infinite loop or exception
nlnwa · Nov 30, 2021 · fe99ad0 · fe99ad0
2 parents 30963d7 + 6c97482
commit fe99ad0
Showing 1 changed file with 2 additions and 5 deletions.
diff --git a/src/main/java/no/nb/nna/veidemann/frontier/db/CrawlQueueWorker.java b/src/main/java/no/nb/nna/veidemann/frontier/db/CrawlQueueWorker.java
@@ -111,12 +111,11 @@ public void run() {
             Error err = ExtraStatusCodes.RUNTIME_EXCEPTION.toFetchError("Timeout waiting for Harvester");
 
             try (JedisContext ctx = JedisContext.forPool(jedisPool)) {
-                String chgId = ctx.getJedis().lpop(CHG_TIMEOUT_KEY);
-                while (chgId != null) {
+                for (String chgId = ctx.getJedis().lpop(CHG_TIMEOUT_KEY); chgId != null; chgId = ctx.getJedis().lpop(CHG_TIMEOUT_KEY)) {
                     try {
                         CrawlHostGroup chg = frontier.getCrawlQueueManager().getCrawlHostGroup(chgId);
                         if (chg.getCurrentUriId().isEmpty()) {
-                            frontier.getCrawlQueueManager().releaseCrawlHostGroup(ctx, chg.getId(), chg.getSessionToken(), 0, true);
+                            frontier.getCrawlQueueManager().releaseCrawlHostGroup(ctx, chgId, chg.getSessionToken(), 0, true);
                             continue;
                         }
                         PostFetchHandler postFetchHandler = new PostFetchHandler(chg, frontier, false);
@@ -125,8 +124,6 @@ public void run() {
                     } catch (Exception e) {
                         LOG.warn("Error while getting chg {}", chgId, e);
                     }
-
-                    chgId = ctx.getJedis().lpop(CHG_TIMEOUT_KEY);
                 }
             } catch (Exception e) {
                 e.printStackTrace();