From 1402f188ef8c14db474d74a535e7407253e0a76d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Wed, 4 Sep 2024 12:31:19 +0000 Subject: [PATCH 1/2] Adopting-ish openZIM bootstrap Getting closer to openZIM bootsrap (we build via setuptools for Cython extension) Fixes #195 - Upgraded build and dev dependencies - Removed setup.cfg (moved metadata to pyproject.toml) - Replaced isort and flake8 with bootstrap's ruff - Added config for pyright (disabled for now as libzim has no type hints) - bootstrap conf for lint/check/test - /!\ using PROFILE by default to have coverage tracing - updated linting to match --- .../workflows/{wheels.yml => CI-wheels.yaml} | 1 - .../workflows/{release.yaml => Publish.yaml} | 0 .github/workflows/QA.yaml | 32 ++ .github/workflows/{test.yml => Tests.yaml} | 38 +-- .gitignore | 223 ++++++++++++- .pre-commit-config.yaml | 27 ++ README.md | 10 +- libzim/libzim.pyx | 10 +- pyproject.toml | 293 +++++++++++++++++- requirements-dev.txt | 8 - setup.cfg | 64 ---- setup.py | 31 +- tasks.py | 188 ++++++----- tests/test_libzim_creator.py | 30 +- tests/test_libzim_reader.py | 12 +- 15 files changed, 735 insertions(+), 232 deletions(-) rename .github/workflows/{wheels.yml => CI-wheels.yaml} (98%) rename .github/workflows/{release.yaml => Publish.yaml} (100%) create mode 100644 .github/workflows/QA.yaml rename .github/workflows/{test.yml => Tests.yaml} (53%) create mode 100644 .pre-commit-config.yaml delete mode 100644 requirements-dev.txt delete mode 100644 setup.cfg diff --git a/.github/workflows/wheels.yml b/.github/workflows/CI-wheels.yaml similarity index 98% rename from .github/workflows/wheels.yml rename to .github/workflows/CI-wheels.yaml index 1676cf92..0b1a1837 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/CI-wheels.yaml @@ -5,7 +5,6 @@ on: push: branches: - main - - windows env: LIBZIM_DL_VERSION: "9.2.3-2" diff --git a/.github/workflows/release.yaml b/.github/workflows/Publish.yaml similarity index 100% rename from .github/workflows/release.yaml rename to .github/workflows/Publish.yaml diff --git a/.github/workflows/QA.yaml b/.github/workflows/QA.yaml new file mode 100644 index 00000000..313c47db --- /dev/null +++ b/.github/workflows/QA.yaml @@ -0,0 +1,32 @@ +name: QA +on: [push] + +env: + LIBZIM_DL_VERSION: "9.2.3-2" + MACOSX_DEPLOYMENT_TARGET: "12.0" + +jobs: + lint: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: "3.12" + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[lint,scripts,test,check] + + - name: Check black formatting + run: inv lint-black + + - name: Check ruff + run: inv lint-ruff + + - name: Check pyright + run: inv check-pyright diff --git a/.github/workflows/test.yml b/.github/workflows/Tests.yaml similarity index 53% rename from .github/workflows/test.yml rename to .github/workflows/Tests.yaml index d1e9d10a..0307cedb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/Tests.yaml @@ -1,28 +1,13 @@ -name: test +name: Tests on: [push] env: LIBZIM_DL_VERSION: "9.2.3-2" MACOSX_DEPLOYMENT_TARGET: "12.0" + # we want cython traces for coverage + PROFILE: "1" jobs: - lint: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 - with: - python-version: "3.12" - architecture: x64 - - - name: Check formatting and linting - run: | - pip install -U invoke - invoke install-dev - invoke check - test: runs-on: ${{ matrix.os }} strategy: @@ -39,25 +24,22 @@ jobs: python-version: ${{ matrix.python }} architecture: x64 - - name: Installing dependencies - run: pip install -U pip setuptools build - - name: Building & installing for tests (with profiling) - env: - PROFILE: 1 - run: pip install -e . + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[test,scripts] - name: move DLLs next to wrapper if: matrix.os == 'windows-2022' run: Move-Item -Force -Path .\libzim\*.dll -Destination .\ - - name: Testing - run: | - pip install pytest pytest-cov cython - py.test --cov=libzim --cov-report=term --cov-report term-missing . + - name: Run the tests + run: inv coverage --args "-vvv" - name: Upload coverage report to codecov if: matrix.os == 'ubuntu-22.04' && matrix.python == '3.11' uses: codecov/codecov-action@v4 with: + fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index a591de75..6ea3576a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,215 @@ -# General cruft +# Created by https://www.toptal.com/developers/gitignore/api/python,macos +# Edit at https://www.toptal.com/developers/gitignore?templates=python,macos + +### macOS ### +# General .DS_Store -.venv -.venv-docker -.mypy_cache -__pycache__ -*.pyc -.tox +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ .coverage -coverage.* -*.swp +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python,macos + +# ignore all vscode, this is not standard configuration in this place +.vscode # Compiled binaries and package builds @@ -32,8 +233,12 @@ libzim/libzim_api.h Pipfile .dev .env +.venv libzim.so libzim.so.* libzim.dylib libzim.*.dylib +zim*.dll +libicu*.dll +zim.lib diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..bc4fa537 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer +- repo: https://github.com/psf/black + rev: "24.8.0" + hooks: + - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.3 + hooks: + - id: ruff +# - repo: https://github.com/RobertCraigie/pyright-python +# rev: v1.1.379 +# hooks: +# - id: pyright +# name: pyright (system) +# description: 'pyright static type checker' +# entry: pyright +# language: system +# 'types_or': [python, pyi] +# require_serial: true +# minimum_pre_commit_version: '2.9.2' diff --git a/README.md b/README.md index 0a4e68b9..42a4cbc7 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,9 @@ Our [PyPI wheels](https://pypi.org/project/libzim/) bundle a [recent release](ht - macOS for `x86_64` and `arm64` - GNU/Linux for `x86_64`, `armhf` and `aarch64` - Linux+musl for `x86_64` and `aarch64` +- Windows for `x64` -Wheels are available for both CPython and PyPy. +Wheels are available for CPython only (but can be built for Pypy). Users on other platforms can install the source distribution (see [Building](#Building) below). @@ -34,10 +35,7 @@ Users on other platforms can install the source distribution (see [Building](#Bu ```sh git clone git@github.com:openzim/python-libzim.git && cd python-libzim -# python -m venv env && source env/bin/activate -pip install -U setuptools invoke -invoke download-libzim install-dev build-ext test -# invoke --list for available development helpers +# hatch run test:coverage ``` See [CONTRIBUTING.md](./CONTRIBUTING.md) for additional details then [Open a ticket](https://github.com/openzim/python-libzim/issues/new) or submit a Pull Request on Github 🤗! @@ -160,7 +158,7 @@ with Creator("test.zim") as creator: | `LIBZIM_DL_VERSION` | `8.1.1` or `2023-04-14` | Specify the C++ libzim binary version to download and bundle. Either a release version string or a date, in which case it downloads a nightly | | `USE_SYSTEM_LIBZIM` | `1` | Uses `LDFLAG` and `CFLAGS` to find the libzim to link against. Resulting wheel won't bundle C++ libzim. | | `DONT_DOWNLOAD_LIBZIM` | `1` | Disable downloading of C++ libzim. Place headers in `include/` and libzim dylib/so in `libzim/` if no using system libzim. It will be bundled in wheel. | -| `PROFILE` | `1` | Enable profile tracing in Cython extension. Required for Cython code coverage reporting. | +| `PROFILE` | `0` | Enable profile tracing in Cython extension. Required for Cython code coverage reporting. | | `SIGN_APPLE` | `1` | Set to sign and notarize the extension for macOS. Requires following informations | | `APPLE_SIGNING_IDENTITY` | `Developer ID Application: OrgName (ID)` | Required for signing on macOS | | `APPLE_SIGNING_KEYCHAIN_PATH` | `/tmp/build.keychain` | Path to the Keychain containing the certificate to sign for macOS with | diff --git a/libzim/libzim.pyx b/libzim/libzim.pyx index 9ab6ebb4..7d2900d5 100644 --- a/libzim/libzim.pyx +++ b/libzim/libzim.pyx @@ -41,7 +41,7 @@ import sys import traceback from collections import OrderedDict from types import ModuleType -from typing import Dict, Generator, Iterator, List, Optional, Set, Tuple, Union +from typing import Dict, Generator, Iterator, List, Optional, Set, TextIO, Tuple, Union from uuid import UUID from cpython.buffer cimport PyBUF_WRITABLE @@ -216,7 +216,7 @@ cdef class WritingBlob: self.ref_content = content self.c_blob = move(zim.Blob( self.ref_content, len(self.ref_content))) - def size(self): + def size(self) -> pyint: return self.c_blob.size() @@ -1151,8 +1151,10 @@ search = searcher.search(query) for path in search.getResults(10, 10) # get result from 10 to 20 (10 results) print(path, archive.get_entry_by_path(path).title)""" search_public_objects = [ + Query, + SearchResultSet, + Search, Searcher, - Query ] search = create_module(search_module_name, search_module_doc, search_public_objects) @@ -1268,7 +1270,7 @@ Usage: print_versions()""" -def print_versions(out: Union[sys.stdout, sys.stderr] = sys.stdout): +def print_versions(out: TextIO = sys.stdout): """print libzim and its dependencies list with their versions""" for library, version in get_versions().items(): prefix = "" if library == "libzim" else "+ " diff --git a/pyproject.toml b/pyproject.toml index 60aacb48..826c6671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,105 @@ [build-system] -requires = [ "setuptools == 68.2.2", "wheel == 0.41.3", "cython == 3.0.5", "delocate==0.12.0" ] +requires = [ + "setuptools == 74.1.1", + "wheel == 0.44.0", + "cython == 3.0.11", + "delocate==0.12.0", +] build-backend = "setuptools.build_meta" -[tool.black] -target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] +[project] +name = "libzim" +version = "3.4.0" +requires-python = ">=3.8,<3.13" +description = "A python-facing API for creating and interacting with ZIM files" +authors = [ + {name = "openZIM", email = "dev@kiwix.org"}, +] +readme = "README.md" +license = {text = "GPL-3.0-or-later"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Topic :: Utilities", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Archiving", + "Topic :: System :: Archiving :: Compression", + "Topic :: System :: Archiving :: Mirroring", + "Topic :: System :: Archiving :: Backup", + "Intended Audience :: Developers", + "Programming Language :: Cython", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Typing :: Stubs Only", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: MacOS", + "Operating System :: POSIX", +] +[project.urls] +Homepage = "https://github.com/openzim/python-libzim" +Donate = "https://www.kiwix.org/en/support-us/" -[tool.pytest.ini_options] -testpaths = ["tests"] -pythonpath = ["."] +[project.optional-dependencies] +scripts = [ + "invoke==2.2.0", +] +lint = [ + "black==24.8.0", + "ruff==0.6.3", + "libzim", + "libzim[build]", +] +check = [ + "pyright==1.1.379", + "libzim", + "libzim[build]", + "libzim[test]", + "types-setuptools", +] +test = [ + "pytest==8.3.2", + "coverage==7.6.1", + # for cython coverage plugin + "libzim[build]", +] +build = [ + "setuptools == 74.1.1", + "wheel == 0.44.0", + "cython == 3.0.11", + "delocate==0.12.0", +] +dev = [ + "pre-commit==3.8.0", + "ipython==8.27.0", + "types-setuptools", + "libzim[scripts]", + "libzim[lint]", + "libzim[test]", + "libzim[check]", + "libzim[build]", +] + +[tool.setuptools] +include-package-data = true +zip-safe = false +packages = [ "libzim" ] + +[tool.setuptools.package-data] +libzim = [ + "libzim.9.dylib", + "libzim.so.9", + "zim-9.dll", + "icuuc74.dll", + "icutu74.dll", + "icuio74.dll", + "icuin74.dll", + "icudt74.dll", +] [tool.cibuildwheel] build = "*" @@ -38,3 +129,193 @@ archs = ["x86_64", "arm64"] # because those python versions were released before our building host version # > requires changing wheel names test-skip = "*_arm64 cp39* cp38*" + +[tool.hatch.build] +exclude = [ + "/.github", +] + +[tool.hatch.build.targets.wheel] +packages = ["libzim"] + + +[tool.hatch.envs.default] +features = ["dev"] + +[tool.hatch.envs.default.scripts] +download-libzim = "inv download-libzim" +build-ext = "inv build-ext" +clean = "inv clean" + +[tool.hatch.envs.test] +features = ["scripts", "test"] + +[tool.hatch.envs.test.env-vars] +PROFILE = "1" + +# not testing on 3.8 as setuptools cannot produce editable wheel +[[tool.hatch.envs.test.matrix]] +python = ["3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.test.scripts] +run = "inv test --args '{args}'" +run-cov = "inv test-cov --args '{args}'" +report-cov = "inv report-cov" +coverage = "inv coverage --args '{args}'" +html = "inv coverage --html --args '{args}'" + +[tool.hatch.envs.lint] +template = "lint" +skip-install = false +features = ["scripts", "lint"] + +[tool.hatch.envs.lint.scripts] +black = "inv lint-black --args '{args}'" +ruff = "inv lint-ruff --args '{args}'" +all = "inv lintall --args '{args}'" +fix-black = "inv fix-black --args '{args}'" +fix-ruff = "inv fix-ruff --args '{args}'" +fixall = "inv fixall --args '{args}'" + +[tool.hatch.envs.check] +features = ["scripts", "check"] + +[tool.hatch.envs.check.scripts] +pyright = "inv check-pyright --args '{args}'" +all = "inv checkall --args '{args}'" + +[tool.black] +line-length = 88 +target-version = ['py312'] + +[tool.ruff] +target-version = "py312" +line-length = 88 +src = ["src"] + +[tool.ruff.lint] +select = [ + "A", # flake8-builtins + # "ANN", # flake8-annotations + "ARG", # flake8-unused-arguments + # "ASYNC", # flake8-async + "B", # flake8-bugbear + # "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "C90", # mccabe + # "COM", # flake8-commas + # "D", # pydocstyle + # "DJ", # flake8-django + "DTZ", # flake8-datetimez + "E", # pycodestyle (default) + "EM", # flake8-errmsg + # "ERA", # eradicate + # "EXE", # flake8-executable + "F", # Pyflakes (default) + # "FA", # flake8-future-annotations + "FBT", # flake8-boolean-trap + # "FLY", # flynt + # "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + # "INP", # flake8-no-pep420 + # "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + # "NPY", # NumPy-specific rules + # "PD", # pandas-vet + # "PGH", # pygrep-hooks + # "PIE", # flake8-pie + # "PL", # Pylint + "PLC", # Pylint: Convention + "PLE", # Pylint: Error + "PLR", # Pylint: Refactor + "PLW", # Pylint: Warning + # "PT", # flake8-pytest-style + # "PTH", # flake8-use-pathlib + # "PYI", # flake8-pyi + "Q", # flake8-quotes + # "RET", # flake8-return + # "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "S", # flake8-bandit + # "SIM", # flake8-simplify + # "SLF", # flake8-self + "T10", # flake8-debugger + "T20", # flake8-print + # "TCH", # flake8-type-checking + # "TD", # flake8-todos + "TID", # flake8-tidy-imports + # "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + # Allow non-abstract empty methods in abstract base classes + "B027", + # Remove flake8-errmsg since we consider they bloat the code and provide limited value + "EM", + # Allow boolean positional values in function calls, like `dict.get(... True)` + "FBT003", + # Ignore checks for possible passwords + "S105", "S106", "S107", + # Ignore warnings on subprocess.run / popen + "S603", + # Ignore complexity + "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", +] +unfixable = [ + # Don't touch unused imports + "F401", +] + +[tool.ruff.lint.isort] +known-first-party = ["libzim"] + +[tool.ruff.lint.flake8-bugbear] +# add exceptions to B008 for fastapi. +extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"] + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.lint.per-file-ignores] +# Tests can use magic values, assertions, and relative imports, and print +"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "ARG002", "N803"] +"setup.py" = ["T201"] +"libzim/*.pyi" = ["FBT001"] + +[tool.pytest.ini_options] +minversion = "7.3" +testpaths = ["tests"] +pythonpath = ["."] + +[tool.coverage] +plugins = "Cython.Coverage" + +[tool.coverage.paths] +libzim = ["libzim"] +tests = ["tests"] + +[tool.coverage.run] +source_pkgs = ["libzim"] +branch = true +parallel = true +omit = [ + "libzim/__about__.py", +] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.pyright] +include = ["libzim", "tests", "tasks.py"] +exclude = [".env/**", ".venv/**", "libzim/libzim.pyi"] +pythonVersion = "3.12" +typeCheckingMode="basic" +disableBytesTypePromotions = true diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 9564bbcb..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,8 +0,0 @@ -isort>=5.10.1,<6.0 -black~=23.11 -flake8>=6.0.0,<7.0 -invoke>=1.7.3,<3.0 -coverage>=6.5.0,<8.0 -pytest>=7.2,<8.0 -pytest-cov>=4.0.0,<5.0 -Cython>=3.0.5 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 76fab60f..00000000 --- a/setup.cfg +++ /dev/null @@ -1,64 +0,0 @@ -[metadata] -name = libzim -version = 3.4.0 -url = https://github.com/openzim/python-libzim -project_urls = - Donate = https://www.kiwix.org/en/support-us/ -description = A python-facing API for creating and interacting with ZIM files -author = Monadical Inc. -author_email = jdc@monadical.com -maintainer = openZIM -maintainer_email = contact+openzim@kiwix.org -long_description = file: README.md -long_description_content_type = text/markdown -license = GPL-3.0-or-later -classifiers = - Development Status :: 5 - Production/Stable - Topic :: Utilities - Topic :: Software Development :: Libraries - Topic :: Software Development :: Libraries :: Python Modules - Topic :: System :: Archiving - Topic :: System :: Archiving :: Compression - Topic :: System :: Archiving :: Mirroring - Topic :: System :: Archiving :: Backup - Intended Audience :: Developers - Programming Language :: Cython - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: 3.12 - Typing :: Stubs Only - License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) - Operating System :: MacOS - Operating System :: POSIX - -[options] -include_package_data = True -zim_safe = False -packages = - libzim -python_requires = - >=3.8,<3.13 -setup_requires = - cython == 3.0.5 -test_requires = - pytest - -[options.package_data] -libzim = - libzim.9.dylib - libzim.so.9 - zim-9.dll - icuuc74.dll - icutu74.dll - icuio74.dll - icuin74.dll - icudt74.dll - -[isort] -profile = black - -[flake8] -max-line-length = 88 diff --git a/setup.py b/setup.py index a3a1451d..774e6a7e 100755 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ class Config: download_libzim: bool = not bool(os.getenv("DONT_DOWNLOAD_LIBZIM") or False) # toggle profiling for coverage report in Cython - profiling: bool = os.getenv("PROFILE", "") == "1" + profiling: bool = os.getenv("PROFILE", "0") == "1" # macOS signing should_sign_apple: bool = bool(os.getenv("SIGN_APPLE") or False) @@ -258,9 +258,10 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: # download a local copy if none present if not fpath.exists(): print(f"> from {url}") - with urllib.request.urlopen(url) as response, open( # nosec # noqa: S310 - fpath, "wb" - ) as fh: # nosec + with ( + urllib.request.urlopen(url) as response, # noqa: S310 + open(fpath, "wb") as fh, # nosec + ): # nosec fh.write(response.read()) else: print(f"> reusing local file {fpath}") @@ -386,16 +387,16 @@ def can_sign_apple(self) -> bool: def get_cython_extension() -> list[Extension]: - define_macros = [] + define_macros: list[tuple[str, str | None]] = [] compiler_directives = {"language_level": "3"} if config.profiling: define_macros += [("CYTHON_TRACE", "1"), ("CYTHON_TRACE_NOGIL", "1")] compiler_directives.update(linetrace="true") - include_dirs = [] - library_dirs = [] - runtime_library_dirs = [] + include_dirs: list[str] = [] + library_dirs: list[str] = [] + runtime_library_dirs: list[str] = [] if config.use_system_libzim: if not config.found_libzim: @@ -426,7 +427,7 @@ def get_cython_extension() -> list[Extension]: if config.platform != "Windows": runtime_library_dirs = ( [f"@loader_path/libzim/{config.libzim_fname}"] - if sysplatform == "Darwin" + if config.platform == "Darwin" else ["$ORIGIN/libzim/"] ) @@ -570,11 +571,9 @@ class DownloadLibzim(Command): user_options = [] # noqa: RUF012 - def initialize_options(self): - ... + def initialize_options(self): ... - def finalize_options(self): - ... + def finalize_options(self): ... def run(self): config.download_to_dest() @@ -583,11 +582,9 @@ def run(self): class LibzimClean(Command): user_options = [] # noqa: RUF012 - def initialize_options(self): - ... + def initialize_options(self): ... - def finalize_options(self): - ... + def finalize_options(self): ... def run(self): config.cleanup() diff --git a/tasks.py b/tasks.py index af786a27..361fd162 100644 --- a/tasks.py +++ b/tasks.py @@ -1,84 +1,130 @@ -#!/usr/bin/env python3 +# pyright: strict, reportUntypedFunctionDecorator=false +import os +from invoke.context import Context +from invoke.tasks import task # pyright: ignore [reportUnknownVariableType] -""" -A description file for invoke (https://www.pyinvoke.org/) -""" +use_pty = not os.getenv("CI", "") -import inspect -# temp local fix for https://github.com/pyinvoke/invoke/issues/891 -if not hasattr(inspect, "getargspec"): - inspect.getargspec = inspect.getfullargspec - -from invoke import task +@task +def clean(ctx: Context): + """download C++ libzim binary""" + ctx.run("python setup.py clean") @task -def download_libzim(c, version=""): +def download_libzim(ctx: Context, version: str = ""): """download C++ libzim binary""" env = f"LIBZIM_DL_VERSION={version}" if version else "" - c.run(f"{env} python setup.py download_libzim") + ctx.run(f"{env} python setup.py download_libzim") @task -def build_ext(c): +def build_ext(ctx: Context): """build extension to use locally (devel, tests)""" - c.run("PROFILE=1 python setup.py build_ext -i") - - -@task -def test(c): - """run test suite""" - c.run("python -m pytest --color=yes --ff -x .") - - -@task -def coverage(c): - """generate coverage report""" - c.run( - "python -m pytest --color=yes " - "--cov=libzim --cov-config=.coveragerc " - "--cov-report=term --cov-report term-missing ." - ) - - -@task -def clean(c): - """remove build folder and generated files""" - c.run("rm -rf build") - c.run("rm -f *.so") - c.run("rm -rf include") - c.run("rm -rf libzim/libzim.{so,dylib} libzim/libzim.so.* libzim/libzim.*.dylib") - - -@task -def install_dev(c): - """install dev requirements""" - c.run("pip install -r requirements-dev.txt") - - -@task -def check(c): - """run Q/A checks""" - c.run("isort --check-only .") - c.run("black --check .") - c.run('echo "one pass for show-stopper syntax errors or undefined names"') - c.run("flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics") - c.run('echo "one pass for small stylistic things"') - c.run("flake8 . --count --statistics") - - -@task -def lint(c): - """Apply Q/A linting""" - c.run("isort .") - c.run("black .") - c.run("flake8 .") - - -if __name__ == "__main__": - print( - "This file is not intended to be directly run.\n" - "Install invoke and run the `invoke` command line." - ) + ctx.run("PROFILE=1 python setup.py build_ext -i") + + +@task(optional=["args"], help={"args": "pytest additional arguments"}) +def test(ctx: Context, args: str = ""): + """run tests (without coverage)""" + build_ext(ctx) + ctx.run(f"pytest {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "pytest additional arguments"}) +def test_cov(ctx: Context, args: str = ""): + """run test vith coverage""" + ctx.run(f"coverage run -m pytest {args}", pty=use_pty) + + +@task(optional=["html"], help={"html": "flag to export html report"}) +def report_cov(ctx: Context, *, html: bool = False): + """report coverage""" + ctx.run("coverage combine", warn=True, pty=use_pty) + ctx.run("coverage report --show-missing", pty=use_pty) + ctx.run("coverage xml", pty=use_pty) + if html: + ctx.run("coverage html", pty=use_pty) + + +@task( + optional=["args", "html"], + help={ + "args": "pytest additional arguments", + "html": "flag to export html report", + }, +) +def coverage(ctx: Context, args: str = "", *, html: bool = False): + """run tests and report coverage""" + test_cov(ctx, args=args) + report_cov(ctx, html=html) + + +@task(optional=["args"], help={"args": "black additional arguments"}) +def lint_black(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("black --version", pty=use_pty) + ctx.run(f"black --check --diff {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def lint_ruff(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff check {args}", pty=use_pty) + + +@task( + optional=["args"], + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, +) +def lintall(ctx: Context, args: str = "."): + """Check linting""" + args = args or "." # needed for hatch script + lint_black(ctx, args) + lint_ruff(ctx, args) + + +@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) +def check_pyright(ctx: Context, args: str = ""): + """check static types with pyright""" + ctx.run("pyright --version") + ctx.run(f"pyright {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) +def checkall(ctx: Context, args: str = ""): + """check static types""" + check_pyright(ctx, args) + + +@task(optional=["args"], help={"args": "black additional arguments"}) +def fix_black(ctx: Context, args: str = "."): + """fix black formatting""" + args = args or "." # needed for hatch script + ctx.run(f"black {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def fix_ruff(ctx: Context, args: str = "."): + """fix all ruff rules""" + args = args or "." # needed for hatch script + ctx.run(f"ruff check --fix {args}", pty=use_pty) + + +@task( + optional=["args"], + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, +) +def fixall(ctx: Context, args: str = "."): + """Fix everything automatically""" + args = args or "." # needed for hatch script + fix_black(ctx, args) + fix_ruff(ctx, args) + lintall(ctx, args) diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index ebf652f0..f29c4c44 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -64,7 +64,7 @@ def fpath(tmpdir): def favicon_data(): return base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQ" - + "ImWO4ISn6HwAE2QIGKsd69QAAAABJRU5ErkJggg==" + "ImWO4ISn6HwAE2QIGKsd69QAAAABJRU5ErkJggg==" ) @@ -129,13 +129,13 @@ def lipsum_item(lipsum): def test_imports(): - assert libzim.writer.Compression # noqa - assert libzim.writer.Blob # noqa - assert libzim.writer.Item # noqa - assert libzim.writer.ContentProvider # noqa - assert libzim.writer.FileProvider # noqa - assert libzim.writer.StringProvider # noqa - assert libzim.writer.Creator # noqa + assert libzim.writer.Compression + assert libzim.writer.Blob + assert libzim.writer.Item + assert libzim.writer.ContentProvider + assert libzim.writer.FileProvider + assert libzim.writer.StringProvider + assert libzim.writer.Creator def test_creator_filename(fpath): @@ -165,6 +165,7 @@ def get_creator_output(fpath, verbose): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + check=False, ) assert ps.returncode == 0 return ps.stdout @@ -251,9 +252,14 @@ def test_creator_nbworkers(fpath, lipsum_item, nb_workers): def test_creator_combine_config(fpath, lipsum_item): - with Creator(fpath).config_verbose(True).config_compression( - "zstd" - ).config_clustersize(1024).config_indexing(True, "eng").config_nbworkers(2) as c: + with ( + Creator(fpath) + .config_verbose(True) + .config_compression("zstd") + .config_clustersize(1024) + .config_indexing(True, "eng") + .config_nbworkers(2) as c + ): c.add_item(lipsum_item) @@ -407,7 +413,7 @@ def test_creator_metadata(fpath, lipsum_item): continue c.add_metadata(name, value) - mdate = datetime.date(*[int(x) for x in metadata.get("Date").split("-")]) + mdate = datetime.date(*[int(x) for x in metadata.get("Date", "").split("-")]) c.add_metadata("Date", mdate) zim = Archive(fpath) diff --git a/tests/test_libzim_reader.py b/tests/test_libzim_reader.py index 3ead9fb8..2fcc458e 100644 --- a/tests/test_libzim_reader.py +++ b/tests/test_libzim_reader.py @@ -129,14 +129,14 @@ "suggestion_count": 1, "suggestion_result": [ "FreedomBox for Communities_Offline Wikipedia " - + "- Wikibooks, open books for an open world.html" + "- Wikibooks, open books for an open world.html" ], "search_string": "main", "search_count": 2, "search_result": [ "Wikibooks.html", "FreedomBox for Communities_Offline Wikipedia " - + "- Wikibooks, open books for an open world.html", + "- Wikibooks, open books for an open world.html", ], "test_path": "FreedomBox for Communities_Offline Wikipedia - Wikibooks, " "open books for an open world.html", @@ -272,7 +272,7 @@ def all_zims(tmpdir_factory): # download libzim tests for url in libzim_urls: - urlretrieve(url, temp_dir / os.path.basename(url)) # nosec + urlretrieve(url, temp_dir / os.path.basename(url)) # noqa: S310 # nosec # create blank using pylibzim creator = libzim.writer.Creator(temp_dir / "blank.zim") @@ -317,10 +317,10 @@ def get_content(): if not entry.is_redirect: _ = entry.get_item().content # Check everything is ok - assert len(content) == 3559 - assert ( + assert content and len(content) == 3559 + assert content and ( bytes(content[:100]) == b'\n\n ' - b'\n That Lucky Old Sun<' # noqa + b'<meta charset="UTF-8">\n <title>That Lucky Old Sun<' ) From 98499fecbdbe94cacbefd15a69cd062003af85c0 Mon Sep 17 00:00:00 2001 From: renaud gaudin <reg@rskg.org> Date: Wed, 4 Sep 2024 14:19:06 +0000 Subject: [PATCH 2/2] Removed support for Python 3.8 (EOL) --- .github/workflows/Tests.yaml | 2 +- CHANGELOG.md | 10 +++++++++- pyproject.toml | 6 ++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 0307cedb..be207103 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [macos-13, windows-2022, ubuntu-22.04] - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c18aa3b..e29ef175 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleazed +## [Unreleased] ### Added - Windows (x64) support (#91) +### Changed + +- Using C++ libzim 9.2.3-2 + +### Removed + +- Support for Python 3.8 (EOL) + ## [3.4.0] - 2023-12-16 ### Added diff --git a/pyproject.toml b/pyproject.toml index 826c6671..61a5034b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "libzim" version = "3.4.0" -requires-python = ">=3.8,<3.13" +requires-python = ">=3.9,<3.13" description = "A python-facing API for creating and interacting with ZIM files" authors = [ {name = "openZIM", email = "dev@kiwix.org"}, @@ -29,7 +29,6 @@ classifiers = [ "Intended Audience :: Developers", "Programming Language :: Cython", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -128,7 +127,7 @@ archs = ["x86_64", "arm64"] # skipping tests on cp <=3.9 as the wheels we produce are on unexpected combinations # because those python versions were released before our building host version # > requires changing wheel names -test-skip = "*_arm64 cp39* cp38*" +test-skip = "*_arm64 cp39*" [tool.hatch.build] exclude = [ @@ -153,7 +152,6 @@ features = ["scripts", "test"] [tool.hatch.envs.test.env-vars] PROFILE = "1" -# not testing on 3.8 as setuptools cannot produce editable wheel [[tool.hatch.envs.test.matrix]] python = ["3.9", "3.10", "3.11", "3.12"]