diff --git a/Makefile b/Makefile index 342c01b..dfea75c 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,12 @@ install-hooks: venv # after running tests .PHONY: test test: - tox + tox -e py37 -- $(TEST_FILE) ifneq ($(strip $(COVERALLS_REPO_TOKEN)),) - .tox/py37/bin/coveralls + .tox/py37/bin/coveralls endif + .PHONY: release-pypi release-pypi: clean autoversion python3 setup.py sdist bdist_wheel diff --git a/README.md b/README.md index 582b8ac..5d3d5ee 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,10 @@ The `tests` directory contains automated tests which you're encouraged to add to (and not break). The `tests-manual` directory contains scripts intended for testing. +To run the a specific test file, run `make tests TEST_FILE=/`. + +For example, to run the `announcements_test.py` file in the `lab` folder, run `make tests TEST_FILE=lab/announcements_test.py` + #### Using pre-commit diff --git a/ocflib/lab/announcements.py b/ocflib/lab/announcements.py new file mode 100644 index 0000000..364bb73 --- /dev/null +++ b/ocflib/lab/announcements.py @@ -0,0 +1,165 @@ +"""Announcements handling""" +from datetime import datetime + +from requests import get +from yaml import safe_load + +# The default branch is main +ANNOUNCEMENTS_URL = ( + 'https://api.github.com/repos/ocf/announcements/contents/{folder}/{id}' +) +# 1 day in seconds +TIME_TO_LIVE = 60 * 60 * 24 + + +class Metadata: + def __init__(self, title, date, author, tags, summary): + self.title = title + self.date = date + self.author = author + self.tags = tags + self.summary = summary + + +class _AnnouncementCache: + def __init__(self) -> None: + # text_cache is a dict of {id: post content} + self.text_cache = {} + # id_cache is a list of ids, ordered by latest to oldest + self.id_cache = [] + self.last_updated = datetime.now() + + def clear_cache(self) -> None: + """Clear the cache if it's too old""" + + if (datetime.now() - self.last_updated).total_seconds() > TIME_TO_LIVE: + self.text_cache.clear() + self.id_cache.clear() + self.last_updated = datetime.now() + + +_announcement_cache_instance = _AnnouncementCache() + + +def _check_id(id: str) -> bool: + """Check if the id is a valid date""" + + try: + datetime.strptime(id, '%Y-%m-%d-%M') + except ValueError: + raise ValueError('Invalid id') + + +def get_all_announcements(folder='announcements') -> [str]: + """ + Get announcements from the announcements repo + The result is a list of IDs from latest to oldest + """ + + posts = get( + url=ANNOUNCEMENTS_URL.format(folder=folder, id=''), + headers={'Accept': 'application/vnd.github+json'}, + ) + posts.raise_for_status() + + res = [] + + for post in posts.json(): + res.append(get_id(post)) + + # Reverse the list so that the order is latest to oldest + res = res[::-1] + + _announcement_cache_instance.id_cache = res + + return res + + +def get_announcement(id: str, folder='announcements') -> str: + """ + Get one particular announcement from the announcements repo + The result is the post content + """ + + _check_id(id) + + if id in _announcement_cache_instance.text_cache: + return _announcement_cache_instance.text_cache[id] + + post = get( + url=ANNOUNCEMENTS_URL.format(folder=folder, id=id + '.md'), + headers={'Accept': 'application/vnd.github.raw'}, + ) + post.raise_for_status() + + _announcement_cache_instance.text_cache[id] = post.text + + return post.text + + +def get_id(post_json: dict) -> str: + """Get announcement id based on the json response""" + + # Since the id is the filename, remove the .md extension + try: + id = post_json['name'][:-3] + except KeyError: + raise KeyError('Missing id in announcement') + + _check_id(id) + + return id + + +def get_metadata(post_text: str) -> Metadata: + """Get the metadata from one announcement""" + + try: + meta_dict = safe_load(post_text.split('---')[1]) + + data = Metadata( + title=meta_dict['title'], + date=meta_dict['date'], + author=meta_dict['author'], + tags=meta_dict['tags'], + summary=meta_dict['summary'], + ) + except (IndexError, KeyError) as e: + raise ValueError(f'Error parsing metadata: {e}') + + return data + + +def get_last_n_announcements(n: int) -> [dict]: + """Get the IDs of last n announcements""" + + assert n > 0, 'n must be positive' + + # check if the cache is too old + _announcement_cache_instance.clear_cache() + + if _announcement_cache_instance.id_cache: + return _announcement_cache_instance.id_cache[:n] + + return get_all_announcements()[:n] + + +def get_last_n_announcements_text(n: int) -> [str]: + """Get the text of last n announcements""" + + assert n > 0, 'n must be positive' + + # check if the cache is too old + _announcement_cache_instance.clear_cache() + + result = [] + + if _announcement_cache_instance.id_cache: + res = _announcement_cache_instance.id_cache[:n] + else: + res = get_all_announcements()[:n] + + for id in res: + result.append(get_announcement(id)) + + return result diff --git a/tests/lab/announcements_test.py b/tests/lab/announcements_test.py new file mode 100644 index 0000000..9459a93 --- /dev/null +++ b/tests/lab/announcements_test.py @@ -0,0 +1,119 @@ +import pytest +from requests.exceptions import HTTPError + +from ocflib.lab.announcements import get_all_announcements +from ocflib.lab.announcements import get_announcement +from ocflib.lab.announcements import get_metadata + +TEST_FOLDER = 'tests' +TEST_IDS = [ + '2002-01-01-00', + '2002-01-01-01', + '2002-01-02-00', + '2023-09-01-00', + '2023-10-01-00', + '2023-11-01-00', +] + + +# scope = module means that the fixture is only run once per module +@pytest.fixture(scope='module') +def get_all() -> [str]: + return get_all_announcements(folder=TEST_FOLDER) + + +# scope = module means that the fixture is only run once per module +@pytest.fixture(scope='module') +def announcement_data(): + # Fetch data once for all tests in this module + return {id: get_announcement(id, folder=TEST_FOLDER) for id in TEST_IDS} + + +# Health check +@pytest.mark.parametrize( + 'id', + TEST_IDS, +) +def test_get_announcement_pass(id): + assert 'testing' in get_announcement(id, folder=TEST_FOLDER) + + +# Those ids are invalid +@pytest.mark.parametrize( + 'id', + [ + '2002-01-00-00212', + '2002-01-01-aa', + '2002-01-02-21a', + '2002-223-02-00', + '2002-01-80-00', + ], +) +def test_get_announcement_bad_id(id): + with pytest.raises(ValueError): + get_announcement(id, folder=TEST_FOLDER) + + +# Those announcements don't exist in the test folder +@pytest.mark.parametrize( + 'id', + [ + '2002-01-01-10', + '2002-01-01-12', + '2002-01-02-30', + ], +) +def test_get_announcement_fail(id): + with pytest.raises(HTTPError): + get_announcement(id, folder=TEST_FOLDER) + + +# Those ids are valid +@pytest.mark.parametrize('id', TEST_IDS) +def test_get_id_pass(id, get_all): + found = False + for post in get_all: + if id == post: + found = True + break + assert found, f'ID {id} not found in announcements' + + +# Those ids don't exist in the test folder +@pytest.mark.parametrize( + 'id', + [ + '2002-01-01-10', + '2002-01-01-12', + '2002-01-02-30', + ], +) +def test_get_id_fail(id, get_all): + for post in get_all: + assert id != post, f'Unexpected ID {id} found in announcements' + + +@pytest.mark.parametrize('id', TEST_IDS) +def test_get_metadata_pass(id, announcement_data): + content = announcement_data[id] + assert 'Victor' == get_metadata(content).author, 'author not found in metadata' + + +def test_get_metadata_missing_metadata(): + content = """ + --- + title: test + date: 2020-01-01 + --- + """ + with pytest.raises(ValueError): + get_metadata(content) + + +def test_get_metadata_bad_format(): + content = """ + title: test + date: 2020-01-01 + """ + with pytest.raises(ValueError): + get_metadata(content) diff --git a/tox.ini b/tox.ini index 80e00ed..7413731 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ venv_update = install= -r {toxinidir}/requirements-dev.txt -e {toxinidir} commands = {[testenv]venv_update} - py.test --cov --cov-report=term-missing -n 1 -v tests -vv + py.test --cov --cov-report=term-missing -n 1 -v tests/{posargs} -vv pre-commit run --all-files [flake8]