diff --git a/CHANGELOG.md b/CHANGELOG.md index 63eb64fe..20d6f453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,8 +28,8 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/ - Fix compatibility with lxml >= 5 which caused the CSS Selector filter to fail (#783 reported by jamesquilty, PR #786 by jamstah) - Fix pep8 test to ignore files in the site-packages directory for cases where the venv is in the project directory (#788 by jamstah) - Fix HTML diff table rendering for long line lengths (#793 by trevorshannon) +- Fix blanks and errors caused by cache load failing when used concurrently (#779 reported by jkmaxwell and others, PR #806 by jamstah) - Fix IndexError after failed edit (#801 by jwilk) -- Fix concurrency issue in Python 3.12 by upgrading to minidb 2.0.8 (fixes #779) ## [2.28] -- 2023-05-03 diff --git a/lib/urlwatch/storage.py b/lib/urlwatch/storage.py index 38d52110..04245061 100644 --- a/lib/urlwatch/storage.py +++ b/lib/urlwatch/storage.py @@ -519,8 +519,8 @@ def load(self, job, guid): return data, timestamp, None, None - def save(self, job, guid, data, timestamp, etag=None): - # Timestamp and ETag are always ignored + def save(self, job, guid, data, timestamp, tries, etag=None): + # Timestamp, tries and ETag are always ignored filename = self._get_filename(guid) with open(filename, 'w+') as fp: fp.write(data) diff --git a/lib/urlwatch/tests/data/disabled-job.yaml b/lib/urlwatch/tests/data/disabled-job.yaml index 8b550c3a..83d7bab9 100644 --- a/lib/urlwatch/tests/data/disabled-job.yaml +++ b/lib/urlwatch/tests/data/disabled-job.yaml @@ -1,6 +1,15 @@ name: "1" -url: "|echo job 1" +command: "echo job 1" enabled: false --- name: "2" -url: "|echo job 2" +command: "echo job 2" +--- +name: "3" +command: "echo job 3" +--- +name: "4" +command: "echo job 4" +--- +name: "5" +command: "echo job 5" diff --git a/lib/urlwatch/tests/test_handler.py b/lib/urlwatch/tests/test_handler.py index 13acb4cb..3fbe7d77 100644 --- a/lib/urlwatch/tests/test_handler.py +++ b/lib/urlwatch/tests/test_handler.py @@ -232,7 +232,7 @@ def test_disabled_job(): urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage) urlwatcher.run_jobs() - assert len(urlwatcher.report.job_states) == 1 + assert len(urlwatcher.report.job_states) == 4 finally: cache_storage.close() diff --git a/lib/urlwatch/tests/test_worker.py b/lib/urlwatch/tests/test_worker.py new file mode 100644 index 00000000..ad8dcc7f --- /dev/null +++ b/lib/urlwatch/tests/test_worker.py @@ -0,0 +1,64 @@ +import sys +from glob import glob + +from urlwatch.jobs import UrlJob, JobBase, ShellJob +from urlwatch.storage import UrlsYaml, UrlsTxt + +import contextlib +import pytest + +import tempfile +import os + +from urlwatch import storage +from urlwatch.config import CommandConfig +from urlwatch.storage import YamlConfigStorage, CacheMiniDBStorage +from urlwatch.main import Urlwatch +from urlwatch.util import import_module_from_source + +root = os.path.join(os.path.dirname(__file__), '..', '..', '..') +here = os.path.dirname(__file__) + + +class ConfigForTest(CommandConfig): + def __init__(self, config, urls, cache, hooks, verbose): + super().__init__([], 'urlwatch', os.path.dirname(__file__), root, config, urls, hooks, cache, verbose) + + +@contextlib.contextmanager +def teardown_func(): + try: + yield + finally: + "tear down test fixtures" + cache = os.path.join(here, 'data', 'cache.db') + if os.path.exists(cache): + os.remove(cache) + + +def test_run_watcher(): + with teardown_func(): + urls = os.path.join(here, 'data', 'disabled-job.yaml') + config = os.path.join(here, 'data', 'urlwatch.yaml') + cache = os.path.join(here, 'data', 'cache.db') + hooks = '' + + config_storage = YamlConfigStorage(config) + urls_storage = UrlsYaml(urls) + cache_storage = CacheMiniDBStorage(cache) + try: + urlwatch_config = ConfigForTest(config, urls, cache, hooks, True) + + # Prime cache + urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage) + urlwatcher.run_jobs() + + # Run multiple times with clean report + for _ in range(100): + urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage) + urlwatcher.run_jobs() + for job_state in urlwatcher.report.job_states: + assert job_state.exception is None, 'Job failed during threading test' + assert job_state.verb == 'unchanged', f'Job verb was "{job_state.verb}" for unchanged output during threading test' + finally: + cache_storage.close()