diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9210cb5..28396f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,15 +7,15 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/psf/black - rev: "24.1.1" + rev: "24.2.0" hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 + rev: v0.2.1 hooks: - id: ruff - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.349 + rev: v1.1.350 hooks: - id: pyright name: pyright (system) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27eb29f..f089347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Metadata hook: add suport for additional-classifiers property #10 + +### Fixed + +- Build hook: fix issue with extract_items when target_path is in a subfolder #11 +- Tests: ensure tests are also ok when ran from a fork or outside any Git structure + ## [0.1.0] - 2024-02-05 ### Added diff --git a/README.md b/README.md index c9ef9f6..145550a 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ This provides a [Hatch](https://pypi.org/project/hatch/)(ling) plugin for common This plugin intentionally has few dependencies, using the Python standard library whenever possible and hence limiting footprint to a minimum. +hatch-openzim adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing). + +hatch-openzim has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.0**. + ## Quick start Assuming you have an openZIM project, you could use such a configuration in your `pyproject.toml` @@ -63,6 +67,7 @@ NOTA: the `dependencies` attribute is not specific to our hook(s), it is a gener | Variable | Required | Description | |---|---|---| | `additional-authors` | N | List of authors that will be appended to the automatic one | +| `additional-classifiers` | N | List of classifiers that will be appended to the automatic ones | | `additional-keywords` | N | List of keywords that will be appended to the automatic ones | | `kind` | N | If set to `scraper`, scrapers keywords will be automatically added as well | | `organization` | N | Override organization (otherwise detected from Github repository to set author and keyword appropriately). Case-insentive. Supported values are `openzim`, `kiwix` and `offspot` | @@ -208,7 +213,7 @@ Some sub-items in the Zip content can be removed afterwards. | `source`| Y | URL of the online ZIP to download | | `zip_paths` | Y | List of relative path in ZIP to select | | `target_paths` | Y | Relative path of the target directory where selected items will be moved (relative to ZIP home folder) | -| `remove` | N | List of glob patterns of ZIP content to remove after extraction (must include the target paths, they are relative to the section `target_dir`) | +| `remove` | N | List of glob patterns of ZIP content to remove after extraction (must include the necessary `target_paths`, they are relative to the section `target_dir`) | | `execute_after` | N | List of shell commands to execute once ZIP extraction is completed; actions are executed with the section `target_dir` as current working directory | Nota: diff --git a/pyproject.toml b/pyproject.toml index 1aaab9f..3983b46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,13 @@ [build-system] -requires = ["hatchling"] +requires = ["hatchling", "hatch-openzim"] build-backend = "hatchling.build" [project] name = "hatch-openzim" -authors = [ - { name = "Kiwix", email = "dev@kiwix.org" }, -] -keywords = ["hatch","plugin","download","file"] requires-python = ">=3.8,<3.13" -description = "Download files at build time" +description = "openZIM hatch plugin to set metadata automatically and download files at build time" readme = "README.md" -license = {text = "GPL-3.0-or-later"} -classifiers = [ +classifiers = [ # needs hatch-openzim 0.2.0 to make it dynamic with additional-classifiers "Framework :: Hatch", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", @@ -25,38 +20,38 @@ classifiers = [ dependencies = [ "hatchling==1.21.1", "packaging==23.2", - "toml==0.10.2", # to be replaced by tomllib once only 3.11 and above is supported + "toml==0.10.2", # to be removed once only 3.11 and above is supported ] -dynamic = ["version"] +dynamic = ["authors", "keywords", "license", "version", "urls"] + +[tool.hatch.metadata.hooks.openzim-metadata] +additional-keywords = ["hatch","plugin","download","file"] +preserve-classifiers = true # to be removed once 0.2.0 is used [project.optional-dependencies] scripts = [ "invoke==2.2.0", ] lint = [ - "black==24.1.1", - "ruff==0.2.0", + "black==24.2.0", + "ruff==0.2.1", ] check = [ - "pyright==1.1.349", + "pyright==1.1.350", ] test = [ "pytest==8.0.0", "coverage==7.4.1", ] dev = [ - "pre-commit==3.6.0", - "debugpy==1.8.0", + "pre-commit==3.6.1", + "debugpy==1.8.1", "hatch-openzim[scripts]", "hatch-openzim[lint]", "hatch-openzim[test]", "hatch-openzim[check]", ] -[project.urls] -Homepage = "https://github.com/openzim/hatch-openzim" -Donate = "https://www.kiwix.org/en/support-us/" - [project.entry-points.hatch] openzim = "hatch_openzim.hooks" @@ -239,4 +234,5 @@ include = ["src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**"] extraPaths = ["src"] pythonVersion = "3.8" -typeCheckingMode="basic" +typeCheckingMode="strict" +disableBytesTypePromotions = true diff --git a/src/hatch_openzim/build_hook.py b/src/hatch_openzim/build_hook.py index d866ba3..fa2813f 100644 --- a/src/hatch_openzim/build_hook.py +++ b/src/hatch_openzim/build_hook.py @@ -1,9 +1,13 @@ +from typing import Any, Dict + from hatchling.builders.hooks.plugin.interface import BuildHookInterface from hatch_openzim.files_install import process as process_files_install -class OpenzimBuildHook(BuildHookInterface): +class OpenzimBuildHook( + BuildHookInterface # pyright: ignore[reportMissingTypeArgument] +): """Hatch build hook to perform custom openzim actions This hook performs: @@ -12,7 +16,7 @@ class OpenzimBuildHook(BuildHookInterface): PLUGIN_NAME = "openzim-build" - def initialize(self, version, build_data): # noqa: ARG002 + def initialize(self, version: str, build_data: Dict[str, Any]): # noqa: ARG002 if "toml-config" in self.config: process_files_install(openzim_toml_location=self.config["toml-config"]) else: diff --git a/src/hatch_openzim/files_install.py b/src/hatch_openzim/files_install.py index 15b98c3..d4b1f24 100644 --- a/src/hatch_openzim/files_install.py +++ b/src/hatch_openzim/files_install.py @@ -7,7 +7,7 @@ from urllib.request import urlopen try: - import tomllib + import tomllib # pyright: ignore[reportMissingTypeStubs] except ImportError: # pragma: no cover import toml as tomllib @@ -190,10 +190,13 @@ def _process_extract_items_action( return with tempfile.TemporaryDirectory() as tempdir: - _extract_zip_from_url(url=source, extract_to=tempdir) + tempath = Path(tempdir) + _extract_zip_from_url(url=source, extract_to=tempath) for index, zip_path in enumerate(zip_paths): - item_src = Path(tempdir) / str(zip_path) + item_src = tempath / str(zip_path) item_dst = base_target_dir / str(target_paths[index]) + if item_dst.parent and not item_dst.parent.exists(): + item_dst.parent.mkdir(parents=True, exist_ok=True) shutil.move(src=str(item_src), dst=item_dst) if "remove" in action_data: @@ -211,7 +214,7 @@ def _remove_items(directory: Path, globs: List[str]): shutil.rmtree(match) -def _download_file(url, download_to): +def _download_file(url: str, download_to: Path): """downloads a file to a given location""" if not url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") @@ -219,7 +222,7 @@ def _download_file(url, download_to): file.write(response.read()) -def _extract_zip_from_url(url, extract_to): +def _extract_zip_from_url(url: str, extract_to: Path): """downloads ZIP from URL and extract in given directory Nota: the ZIP is temporarily saved on disk (there is no convenient function diff --git a/src/hatch_openzim/metadata.py b/src/hatch_openzim/metadata.py index 9530f37..d3406cc 100644 --- a/src/hatch_openzim/metadata.py +++ b/src/hatch_openzim/metadata.py @@ -1,9 +1,10 @@ from pathlib import Path +from typing import Any, Dict from hatch_openzim.utils import get_github_info, get_python_versions -def update(root: str, config: dict, metadata: dict): +def update(root: str, config: Dict[str, Any], metadata: Dict[str, Any]): """Update the project table's metadata.""" # Check for absence of metadata we will set + presence in the dynamic property @@ -60,4 +61,5 @@ def update(root: str, config: dict, metadata: dict): ] for python_version in get_python_versions(metadata["requires-python"]): classifiers.append(f"Programming Language :: Python :: {python_version}") + classifiers.extend(config.get("additional-classifiers", [])) metadata["classifiers"] = classifiers diff --git a/src/hatch_openzim/metadata_hook.py b/src/hatch_openzim/metadata_hook.py index c332bf4..fb86bc5 100644 --- a/src/hatch_openzim/metadata_hook.py +++ b/src/hatch_openzim/metadata_hook.py @@ -1,4 +1,4 @@ -from __future__ import annotations +from typing import Any, Dict from hatchling.metadata.plugin.interface import MetadataHookInterface @@ -14,6 +14,10 @@ class OpenzimMetadataHook(MetadataHookInterface): PLUGIN_NAME = "openzim-metadata" - def update(self, metadata: dict): + def update(self, metadata: Dict[str, Any]): """Update the project table's metadata.""" - update(root=self.root, config=self.config, metadata=metadata) + update( + root=self.root, + config=self.config, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] + metadata=metadata, + ) diff --git a/src/hatch_openzim/utils.py b/src/hatch_openzim/utils.py index 7cfab79..51ca1eb 100644 --- a/src/hatch_openzim/utils.py +++ b/src/hatch_openzim/utils.py @@ -1,8 +1,7 @@ import configparser import re -from collections import namedtuple from pathlib import Path -from typing import List +from typing import List, NamedTuple, Optional from packaging.specifiers import SpecifierSet from packaging.version import Version @@ -14,7 +13,12 @@ r"""(?P.*?)(?:.git)?$""" ) -GithubInfo = namedtuple("GithubInfo", ["homepage", "organization", "repository"]) + +class GithubInfo(NamedTuple): + homepage: str + organization: Optional[str] + repository: Optional[str] + DEFAULT_GITHUB_INFO = GithubInfo( homepage="https://www.kiwix.org", organization=None, repository=None @@ -57,8 +61,8 @@ def get_python_versions(requires_python: str) -> List[str]: last_py1_minor = 6 last_py2_minor = 7 - major_versions = [] - minor_versions = [] + major_versions: list[str] = [] + minor_versions: list[str] = [] for major in range(1, 10): # this will work up to Python 10 ... major_added = False last_minor = 100 # this supposes we will never have Python x.100 diff --git a/tests/configs/full.toml b/tests/configs/full.toml index 6acd346..eb230c5 100644 --- a/tests/configs/full.toml +++ b/tests/configs/full.toml @@ -38,6 +38,12 @@ source="https://tmp.kiwix.org/ci/hatch_openzim_testsets/testset2.zip" zip_paths=["keep1/file1.txt"] target_paths=["file123.txt"] +[files.part2.actions.action5] +action="extract_items" +source="https://tmp.kiwix.org/ci/hatch_openzim_testsets/testset2.zip" +zip_paths=["keep1/file1.txt"] +target_paths=["action5/subfolder1/file123.txt"] + # part without any actions [files.part3.config] target_dir="part3" diff --git a/tests/configs/gitconfig b/tests/configs/gitconfig new file mode 100644 index 0000000..43325bd --- /dev/null +++ b/tests/configs/gitconfig @@ -0,0 +1,11 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true +[remote "origin"] + url = git@github.com:openzim/hatch-openzim.git + fetch = +refs/heads/*:refs/remotes/origin/* +[branch "main"] + remote = origin + merge = refs/heads/main diff --git a/tests/test_files_install.py b/tests/test_files_install.py index a045ee9..fb08dde 100644 --- a/tests/test_files_install.py +++ b/tests/test_files_install.py @@ -20,6 +20,7 @@ def nominal_files(): "part1/somewhere/something.txt", "part1/somewhere_else/something.txt", "part2/file123.txt", + "part2/action5/subfolder1/file123.txt", "part2/action2/file1.txt", "part2/action2/file2.txt", "part2/action3/file1.json", @@ -52,7 +53,7 @@ def test_no_arg(): ("other_stuff.toml"), ], ) -def test_ignored_silently(config_file): +def test_ignored_silently(config_file: str): """Test cases where the config file is passed but there is no relevant content""" files_install.process( str((Path(__file__).parent / "configs" / config_file).absolute()) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 77c4410..acaae72 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,13 +1,17 @@ import os +import shutil from pathlib import Path +from typing import Any, Dict, List, Union import pytest from hatch_openzim.metadata import update +Metadata = Dict[str, Union[str, List[str]]] + @pytest.fixture -def dynamic_metadata(): +def dynamic_metadata() -> List[str]: return [ "authors", "classifiers", @@ -18,16 +22,39 @@ def dynamic_metadata(): @pytest.fixture -def metadata(dynamic_metadata): +def root_folder(tmp_path: Path) -> str: + """ + Returns a "virtual" root folder with a "virtual" git config + + Git config comes from the tests/configs/gitconfig file + + This is necessary to ensure tests run always with the same git configuration file, + to avoid variability coming from: + - tests ran on plain files (not linked to any git repo) + - tests ran on a repository fork (e.g myuser/hatch-openzim) + - tests ran with a different remote (nothing forces main remote to be named origin) + """ + git_folder = tmp_path / ".git" + git_folder.mkdir() + shutil.copy( + Path(os.path.dirname(os.path.abspath(__file__))).parent + / "tests/configs/gitconfig", + git_folder / "config", + ) + return str(tmp_path) + + +@pytest.fixture +def metadata(dynamic_metadata: List[str]) -> Metadata: return { "requires-python": ">=3.10,<3.12", "dynamic": dynamic_metadata, } -def test_metadata_nominal(metadata): +def test_metadata_nominal(metadata: Metadata, root_folder: str): update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config={}, metadata=metadata, ) @@ -57,7 +84,11 @@ def test_metadata_nominal(metadata): ("urls"), ], ) -def test_metadata_missing_dynamic(metadata, metadata_key): +def test_metadata_missing_dynamic( + metadata: Metadata, metadata_key: str, root_folder: str +): + assert isinstance(metadata["dynamic"], List) + assert all(isinstance(item, str) for item in metadata["dynamic"]) metadata["dynamic"].remove(metadata_key) with pytest.raises( Exception, @@ -65,7 +96,7 @@ def test_metadata_missing_dynamic(metadata, metadata_key): " metadata hook", ): update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config={}, metadata=metadata, ) @@ -81,7 +112,9 @@ def test_metadata_missing_dynamic(metadata, metadata_key): ("urls"), ], ) -def test_metadata_metadata_already_there(metadata, metadata_key): +def test_metadata_metadata_already_there( + metadata: Metadata, metadata_key: str, root_folder: str +): metadata[metadata_key] = "some_value" with pytest.raises( Exception, @@ -89,7 +122,7 @@ def test_metadata_metadata_already_there(metadata, metadata_key): "openzim metadata hook", ): update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config={}, metadata=metadata, ) @@ -105,23 +138,25 @@ def test_metadata_metadata_already_there(metadata, metadata_key): ("urls"), ], ) -def test_metadata_preserve_value(metadata, metadata_key): +def test_metadata_preserve_value( + metadata: Metadata, metadata_key: str, root_folder: str +): metadata[metadata_key] = f"some_value_for_{metadata_key}" - config = {} + config: Dict[str, Any] = {} config[f"preserve-{metadata_key}"] = True update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config=config, metadata=metadata, ) assert metadata[metadata_key] == f"some_value_for_{metadata_key}" -def test_metadata_additional_keywords(metadata): - config = {} +def test_metadata_additional_keywords(metadata: Metadata, root_folder: str): + config: Dict[str, Any] = {} config["additional-keywords"] = ["keyword1", "keyword2"] update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config=config, metadata=metadata, ) @@ -129,11 +164,33 @@ def test_metadata_additional_keywords(metadata): assert set(metadata["keywords"]) == {"openzim", "keyword1", "keyword2"} -def test_metadata_additional_authors(metadata): - config = {} +def test_metadata_additional_classifiers(metadata: Metadata, root_folder: str): + config: Dict[str, Any] = {} + config["additional-classifiers"] = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + ] + update( + root=root_folder, + config=config, + metadata=metadata, + ) + # we compare sets because order is not relevant + assert set(metadata["classifiers"]) == { + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + } + + +def test_metadata_additional_authors(metadata: Metadata, root_folder: str): + config: Dict[str, Any] = {} config["additional-authors"] = [{"email": "someone@acme.org", "name": "Some One"}] update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config=config, metadata=metadata, ) @@ -156,12 +213,14 @@ def test_metadata_additional_authors(metadata): (None, "openzim"), ], ) -def test_metadata_organization(organization, expected_result, metadata): - config = {} +def test_metadata_organization( + organization: str, expected_result: str, metadata: Metadata, root_folder: str +): + config: Dict[str, Any] = {} if organization: config["organization"] = organization update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config=config, metadata=metadata, ) @@ -175,11 +234,11 @@ def test_metadata_organization(organization, expected_result, metadata): raise Exception(f"Unexpected expected result: {expected_result}") -def test_metadata_is_scraper(metadata): - config = {} +def test_metadata_is_scraper(metadata: Metadata, root_folder: str): + config: Dict[str, Any] = {} config["kind"] = "scraper" update( - root=str(Path(os.path.dirname(os.path.abspath(__file__))).parent), + root=root_folder, config=config, metadata=metadata, ) diff --git a/tests/test_utils.py b/tests/test_utils.py index f0ce928..8a207eb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import tempfile from contextlib import contextmanager from pathlib import Path -from typing import List +from typing import Any, Callable, Generator, List import pytest @@ -9,7 +9,7 @@ @pytest.fixture -def mock_git_config(): +def mock_git_config() -> Generator[Callable[[str, str], Any], None, None]: @contextmanager def _mock_git_config(git_origin_url: str, remote_name: str = "origin"): with tempfile.NamedTemporaryFile() as temp_file: @@ -55,11 +55,11 @@ def _mock_git_config(git_origin_url: str, remote_name: str = "origin"): ], ) def test_get_github_project_homepage_valid_url( - mock_git_config, - git_url, - expected_homepage_url, - expected_organization, - expected_repository, + mock_git_config: Callable[[str], Any], + git_url: str, + expected_homepage_url: str, + expected_organization: str, + expected_repository: str, ): with mock_git_config(git_url) as git_config_path: assert get_github_info(git_config_path=git_config_path) == GithubInfo( @@ -69,7 +69,7 @@ def test_get_github_project_homepage_valid_url( ) -def test_get_github_project_homepage_invalid_url(mock_git_config): +def test_get_github_project_homepage_invalid_url(mock_git_config: Callable[[str], Any]): # Test the function with an invalid URL with mock_git_config("http://github.com/oneuser/onerepo.git") as git_config_path: assert get_github_info(git_config_path=git_config_path) == GithubInfo( @@ -84,10 +84,12 @@ def test_get_github_project_missing_git_config(): ) -def test_get_github_project_homepage_invalid_remote(mock_git_config): +def test_get_github_project_homepage_invalid_remote( + mock_git_config: Callable[[str, str], Any], +): # Test the function with an invalid URL with mock_git_config( - "https://github.com/oneuser/onerepo.git", remote_name="origin2" + "https://github.com/oneuser/onerepo.git", "origin2" ) as git_config_path: assert get_github_info(git_config_path=git_config_path) == GithubInfo( homepage="https://www.kiwix.org", organization=None, repository=None