Add tests that exercise threading

Threading was the cause of the issue in #779, so adding a test that will
exercise multi-threading.

CHANGELOG.md

@@ -28,8 +28,8 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/
 - Fix compatibility with lxml >= 5 which caused the CSS Selector filter to fail (#783 reported by jamesquilty, PR #786 by jamstah)
 - Fix pep8 test to ignore files in the site-packages directory for cases where the venv is in the project directory (#788 by jamstah)
 - Fix HTML diff table rendering for long line lengths (#793 by trevorshannon)
+- Fix blanks and errors caused by cache load failing when used concurrently (#779 reported by jkmaxwell and others, PR #806 by jamstah)
 - Fix IndexError after failed edit (#801 by jwilk)
-- Fix concurrency issue in Python 3.12 by upgrading to minidb 2.0.8 (fixes #779)
 
 ## [2.28] -- 2023-05-03
 

lib/urlwatch/storage.py

@@ -519,8 +519,8 @@ def load(self, job, guid):
 
         return data, timestamp, None, None
 
-    def save(self, job, guid, data, timestamp, etag=None):
-        # Timestamp and ETag are always ignored
+    def save(self, job, guid, data, timestamp, tries, etag=None):
+        # Timestamp, tries and ETag are always ignored
         filename = self._get_filename(guid)
         with open(filename, 'w+') as fp:
             fp.write(data)

lib/urlwatch/tests/data/disabled-job.yaml

@@ -1,6 +1,15 @@
 name: "1"
-url: "|echo job 1"
+command: "echo job 1"
 enabled: false
 ---
 name: "2"
-url: "|echo job 2"
+command: "echo job 2"
+---
+name: "3"
+command: "echo job 3"
+---
+name: "4"
+command: "echo job 4"
+---
+name: "5"
+command: "echo job 5"

lib/urlwatch/tests/test_handler.py

@@ -232,7 +232,7 @@ def test_disabled_job():
             urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage)
             urlwatcher.run_jobs()
 
-            assert len(urlwatcher.report.job_states) == 1
+            assert len(urlwatcher.report.job_states) == 4
         finally:
             cache_storage.close()
 

lib/urlwatch/tests/test_worker.py

@@ -0,0 +1,64 @@
+import sys
+from glob import glob
+
+from urlwatch.jobs import UrlJob, JobBase, ShellJob
+from urlwatch.storage import UrlsYaml, UrlsTxt
+
+import contextlib
+import pytest
+
+import tempfile
+import os
+
+from urlwatch import storage
+from urlwatch.config import CommandConfig
+from urlwatch.storage import YamlConfigStorage, CacheMiniDBStorage
+from urlwatch.main import Urlwatch
+from urlwatch.util import import_module_from_source
+
+root = os.path.join(os.path.dirname(__file__), '..', '..', '..')
+here = os.path.dirname(__file__)
+
+
+class ConfigForTest(CommandConfig):
+    def __init__(self, config, urls, cache, hooks, verbose):
+        super().__init__([], 'urlwatch', os.path.dirname(__file__), root, config, urls, hooks, cache, verbose)
+
+
+@contextlib.contextmanager
+def teardown_func():
+    try:
+        yield
+    finally:
+        "tear down test fixtures"
+        cache = os.path.join(here, 'data', 'cache.db')
+        if os.path.exists(cache):
+            os.remove(cache)
+
+
+def test_run_watcher():
+    with teardown_func():
+        urls = os.path.join(here, 'data', 'disabled-job.yaml')
+        config = os.path.join(here, 'data', 'urlwatch.yaml')
+        cache = os.path.join(here, 'data', 'cache.db')
+        hooks = ''
+
+        config_storage = YamlConfigStorage(config)
+        urls_storage = UrlsYaml(urls)
+        cache_storage = CacheMiniDBStorage(cache)
+        try:
+            urlwatch_config = ConfigForTest(config, urls, cache, hooks, True)
+
+            # Prime cache
+            urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage)
+            urlwatcher.run_jobs()
+
+            # Run multiple times with clean report
+            for _ in range(100):
+                urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage)
+                urlwatcher.run_jobs()
+                for job_state in urlwatcher.report.job_states:
+                    assert job_state.exception is None, 'Job failed during threading test'
+                    assert job_state.verb == 'unchanged', f'Job verb was "{job_state.verb}" for unchanged output during threading test'
+        finally:
+            cache_storage.close()