diff --git a/.github/workflows/Publish.yaml b/.github/workflows/Publish.yaml new file mode 100644 index 0000000..f5c57b6 --- /dev/null +++ b/.github/workflows/Publish.yaml @@ -0,0 +1,60 @@ +name: Build and upload to PyPI + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-22.04 + permissions: + id-token: write # mandatory for PyPI trusted publishing + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: pyproject.toml + architecture: x64 + + - uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: install handlebars + run: | + npm install -g handlebars + + - name: Build packages + run: | + pip install -U pip build + python -m build --sdist --wheel + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1.8 + # dont specify anything for Trusted Publishing + # https://docs.pypi.org/trusted-publishers + # with: + # # Using token + # user: __token__ + # password: ${{ secrets.PYPI_API_TOKEN }} + # + # # Using token on test index + # password: ${{ secrets.PYPI_TEST_API_TOKEN }} + # repository_url: https://test.pypi.org/legacy/ + + - name: Build and push Docker image + uses: openzim/docker-publish-action@v10 + with: + image-name: openzim/nautilus + tag-pattern: /^v([0-9.]+)$/ + latest-on-tag: true + restrict-to: openzim/nautilus + registries: ghcr.io + credentials: + GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} + repo_description: auto + repo_overview: auto diff --git a/.github/workflows/docker.yml b/.github/workflows/PublishDockerDevImage.yaml similarity index 64% rename from .github/workflows/docker.yml rename to .github/workflows/PublishDockerDevImage.yaml index 5746fdb..37e665d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/PublishDockerDevImage.yaml @@ -1,25 +1,23 @@ -name: Docker +name: Publish Docker dev image on: push: branches: - main - tags: - - v* jobs: - build-and-push: - name: Deploy Docker Image + publish: runs-on: ubuntu-22.04 + steps: - - uses: actions/checkout@v3.4.0 - - name: Build and push + - uses: actions/checkout@v3 + + - name: Build and push Docker image uses: openzim/docker-publish-action@v10 with: image-name: openzim/nautilus - on-master: dev - tag-pattern: /^v([0-9.]+)$/ - latest-on-tag: true + manual-tag: dev + latest-on-tag: false restrict-to: openzim/nautilus registries: ghcr.io credentials: diff --git a/.github/workflows/QA.yaml b/.github/workflows/QA.yaml new file mode 100644 index 0000000..48ccee5 --- /dev/null +++ b/.github/workflows/QA.yaml @@ -0,0 +1,34 @@ +name: QA + +on: + pull_request: + push: + branches: + - main + +jobs: + check-qa: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[lint,scripts,test,check] + + - name: Check black formatting + run: inv lint-black + + - name: Check ruff + run: inv lint-ruff + + - name: Check pyright + run: inv check-pyright diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml new file mode 100644 index 0000000..f388380 --- /dev/null +++ b/.github/workflows/Tests.yaml @@ -0,0 +1,75 @@ +name: Tests + +on: + pull_request: + push: + branches: + - main + +jobs: + run-tests: + strategy: + matrix: + os: [ubuntu-22.04] + python: ["3.11", "3.12"] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[test,scripts] + + - name: Run the tests + run: inv coverage --args "-vvv" + + - name: Upload coverage report to codecov + if: matrix.python == '3.12' + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + build_python: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Ensure we can build Python targets + run: | + pip install -U pip build + python3 -m build --sdist --wheel + + build_docker: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Ensure we can build the Docker image + run: | + docker build -t testimage . + + - name: Ensure we can start the Docker image + run: | + docker run --rm testimage + + # OR if image is a daemon process + # - name: Ensure we can start the Docker image + # run: | + # docker run --rm testimage + # sleep 5 + # docker ps | grep test_container + # docker stop test_container diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml deleted file mode 100644 index c4480a9..0000000 --- a/.github/workflows/qa.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: QA - -on: [push] - -env: - # black default - MAX_LINE_LENGTH: 88 - -jobs: - check-qa: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v3 - with: - python-version: "3.11" - architecture: x64 - - - name: Check black formatting - run: | - pip install black==23.3.0 - black --version - black --check . - - - name: Check flake8 linting - run: | - pip install flake8==6.0.0 - flake8 --version - flake8 . --count --max-line-length=$MAX_LINE_LENGTH --statistics - - - name: Check import order with isort - run: | - pip install isort==5.12.0 - isort --version - isort --profile black --check . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 45f0ae3..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: release -on: - release: - types: [published] - tags: - - v* - -env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - -jobs: - release: - environment: release - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: x64 - - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: install handlebars - run: | - npm install -g handlebars - - - name: Build sdist and wheel - run: | - pip install --upgrade pip build wheel - python3 -m build - - - name: Push release to PyPI - if: github.event_name == 'release' - run: | - pip install --upgrade twine - twine check dist/* - twine upload dist/* diff --git a/.gitignore b/.gitignore index 288c03e..062a40a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,16 @@ +src/nautiluszim/templates/vendors/* +precompiled.js +output/ +input/ +contrib/ +*.orig +node_modules +package-lock.json + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -20,7 +33,6 @@ parts/ sdist/ var/ wheels/ -pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -50,6 +62,11 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +cover/ + +# Translations +*.mo +*.pot # Django stuff: *.log @@ -68,6 +85,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -78,7 +96,9 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -87,7 +107,22 @@ ipython_config.py # install all needed dependencies. #Pipfile.lock -# PEP 582; used by e.g. github.com/David-OConnor/pyflow +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff @@ -124,5 +159,27 @@ dmypy.json # Pyre type checker .pyre/ -nautiluszim/templates/vendors/* -precompiled.js +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8c653e7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer +- repo: https://github.com/psf/black + rev: "24.4.2" + hooks: + - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.4 + hooks: + - id: ruff +- repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.362 + hooks: + - id: pyright + name: pyright (system) + description: 'pyright static type checker' + entry: pyright + language: system + 'types_or': [python, pyi] + require_serial: true + minimum_pre_commit_version: '2.9.2' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index c663cc5..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,81 +0,0 @@ -# nautilus devel - - -## setup - -__Note__: make sure you've installed non-python dependencies first, as mentioned in the [README](README.md). - -Then, setup a python `virtualenv` using `python 3.6+` and install our python dependencies using `pip`: - -```bash -curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -python get-pip.py # you may use sudo -pip install -U virtualenv # sudo to install globally -virtualenv -p python3.6 nautilus-env -source nautilus-env/bin/activate -pip install -r requirements.txt -``` - -You now have an isolated python installation in the `./nautilus-env/` folder, containing all the python dependencies. - -__Note__: you must activate the `virtualenv` before running the scraper so it uses this virtual python installation - -``` bash -source nautilus-env/bin/activate -``` - -and then run it: - -``` bash -python nautiluszim --help -# or, without activating the virtualenv: -nautilus-env/bin/python nautiluszim --help -``` - -_Note_: when using raw source, you need to compile JS templates first (see bellow). - -## contributions - -* Open issues, bug reports and send PRs [on github](https://github.com/openzim/nautilus). -* Make sure it's `py3.6+` compatible. -* Use [black](https://github.com/psf/black) code formatting. - -#### templates - -In-JS templates uses [handlebars](https://handlebarsjs.com). When editing them, always recompile them all into precompiled.js: - -``` -handlebars nautiluszim/templates -f nautiluszim/templates/precompiled.js -``` - -## notes - -In order to support all platform for audio/video playback, we use `ogv.js`, to play ogv/ogg files on browsers that don't support it. Using `video.js`, we default to native playback if supported. - -`ogv.js` is an emscripten-based JS decoder for webm and thus dynamically loads differents parts at run-time on platforms that needs them. It has two consequences: - -* `file://` scheme doesn't work as the binary `.wasm` files are sent naively as `text/html` instead of `application/oct-stream`. If you want to use the HTML generated folder instead of ZIM, serve it through a web server and [configure the Content-Type response](https://emscripten.org/docs/compiling/WebAssembly.html#web-server-setup). -* [ZIM](https://wiki.openzim.org/wiki/ZIM_file_format) places JS files under the `/-/` namespace and binary ones under the `/I/` one. Dynamically loading of JS and WASM files within `ogv.js` requires us to tweak it to introduce some ZIM-specific logic. See `fix_ogvjs_dist.py`. - - -## i18n - -`nautilus` has very minimal non-content text but still uses gettext through [babel](http://babel.pocoo.org/en/latest/) to internationalize. - -To add a new locale (`fr` in this example, use only ISO-639-1): - -1. init for your locale: `pybabel init -d nautiluszim/locale -l fr -i nautiluszim/locale/messages.pot` -2. make sure the POT is up to date `pybabel extract -o nautiluszim/locale/messages.pot nautiluszim` -3. update your locale's catalog `pybabel update -d nautiluszim/locale/ -l fr -i nautiluszim/locale/messages.pot` -3. translate the PO file ([poedit](https://poedit.net/) is your friend) -4. compile updated translation `pybabel compile -d nautiluszim/locale -l fr` - -## releasing - -* Update your dependencies: `pip install -U setuptools wheel twine` -* Make sure CHANGELOG is up-to-date -* Bump version on `nautiluszim/VERSION` -* Build packages `python ./setup.py sdist bdist_wheel` -* Upload to PyPI `twine upload dist/nautiluszim-1.0.0*`. -* Commit your CHANGELOG + version bump changes -* Tag version on git `git tag -a v1.0.0` diff --git a/Dockerfile b/Dockerfile index e6a059d..2e0b806 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,32 @@ -FROM python:3.8 +FROM python:3.11-slim-bookworm +LABEL org.opencontainers.image.source https://github.com/openzim/nautilus # Install necessary packages -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends locales-all unzip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -RUN wget -nv -L https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-x64.tar.xz && \ - tar -C /usr/local --strip-components 1 -xf node-v12.16.3-linux-x64.tar.xz && \ - rm node-v12.16.3-linux-x64.tar.xz && npm install -g handlebars +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + # locales required if tool has any i18n support + locales-all wget \ + && rm -rf /var/lib/apt/lists/* \ + && python -m pip install --no-cache-dir -U \ + pip \ +&& wget -nv -L https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-x64.tar.gz \ +&& tar -C /usr/local --strip-components 1 -xf node-v12.16.3-linux-x64.tar.gz \ +&& rm node-v12.16.3-linux-x64.tar.gz \ +&& npm install -g handlebars -COPY nautiluszim /src/nautiluszim -COPY get_js_deps.sh requirements.txt setup.py README.md LICENSE MANIFEST.in /src/ -RUN pip3 install --no-cache-dir -r /src/requirements.txt && cd /src/ && python3 ./setup.py install +# Copy pyproject.toml and its dependencies +COPY pyproject.toml README.md /src/ +COPY src/nautiluszim/__about__.py /src/src/nautiluszim/__about__.py + +# Install Python dependencies +RUN pip install --no-cache-dir /src + +# Copy code + associated artifacts +COPY src /src/src +COPY *.md /src/ + +# Install + cleanup +RUN pip install --no-cache-dir /src \ + && rm -rf /src CMD ["nautiluszim", "--help"] diff --git a/README.md b/README.md index f217432..e9725e0 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,15 @@ -Nautilus -============= + +# Nautilus + +`nautilus` turns a collection of documents into a browsable [ZIM file](https://openzim.org). [![CodeFactor](https://www.codefactor.io/repository/github/openzim/nautilus/badge)](https://www.codefactor.io/repository/github/openzim/nautilus) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) +[![codecov](https://codecov.io/gh/openzim/nautilus/branch/main/graph/badge.svg)](https://codecov.io/gh/openzim/nautilus) [![PyPI version shields.io](https://img.shields.io/pypi/v/nautiluszim.svg)](https://pypi.org/project/nautiluszim/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/nautiluszim.svg)](https://pypi.org/project/nautiluszim) [![Docker](https://ghcr-badge.deta.dev/openzim/nautilus/latest_tag?label=docker)](https://ghcr.io/openzim/nautilus) -`nautilus` turns a collection of documents into a browsable [ZIM file](https://openzim.org). - It downloads the video (webm or mp4 format – optionally recompress it in lower-quality, smaller size), the thumbnails, the subtitles and the authors' profile pictures ; then, it creates a static HTML files folder of it before creating a ZIM off of it. # Preparing the archive @@ -67,50 +69,34 @@ Either inside the archive ZIP as `/about.html` or elsewhere, specified via `--ab * Use inline styling if required, but no styling is recommended. * Include one logo inline if required. -# Requirements - -* `wget` and `unzip` to install JS dependencies. See `get_js_deps.sh` if you want to do it manually. -* `wget` is used to download archive files as well. -* [`handlebars`](https://handlebarsjs.com) is used to compile JS templates - -# Installation - -`nautilus` is a python program. if you are not using the docker image, you are advised to use it in a virtual-environment. See `requirements.txt` for the list of python dependencies. - -## docker +## Usage ```sh -docker run -v my_dir:/output ghcr.io/openzim/nautilus nautiluszim --help -``` +❯ nautuluszim --help +usage: nautuluszim [-h] [-V] -## pip +# everything bundled in a ZIP +nautiluszim --archive my-content.zip -```sh -pip install nautiluszim -nautiluszim --help +# In this mode every file entry must have a valid url. +nautiluszim --collection https://example.com/to-your-collection-file ``` -## develop +### Installation -Nautilus relies on downloaded and pre-compiled assets. Make sure to install as editable to grab them. +You'd want to install it in a dedicated virtual-environment (`python3 -m venv some-env && source ./some-env/bin/activate`) ```sh -pip install -e . -nautiluszim --help +❯ pip install nautiluszim ``` -# Usage +### Contributing ```sh -nautiluszim --archive my-content.zip -``` -Or -```sh -nautiluszim --collection https://example.com/to-your-collection-file -# In this mode every file entry must have a valid url. +❯ pip install -e . ``` -## Notes +#### Notes * On macOS, the locale setting is buggy. You need to launch it with the `LANGUAGE` environment variable (as ISO-639-1) for the translations to work. @@ -118,6 +104,6 @@ nautiluszim --collection https://example.com/to-your-collection-file LANGUAGE=fr nautiluszim --language fra ``` -## Development +Nautilus adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing). -See [CONTRIBUTING.md](CONTRIBUTING.md) +Nautilus has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.0**. diff --git a/get_js_deps.sh b/get_js_deps.sh deleted file mode 100755 index b6b0d43..0000000 --- a/get_js_deps.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh - -set -e # fail script on error - -### -# download JS dependencies and place them in our templates/assets folder -# then launch our ogv.js script to fix dynamic loading links -### - -if ! command -v wget > /dev/null; then - echo "you need wget." - exit 1 -fi - -if ! command -v unzip > /dev/null; then - echo "you need unzip." - exit 1 -fi - -# Absolute path this script is in. -SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )" -VENDORS_PATH="${SCRIPT_PATH}/nautiluszim/templates/vendors" - -echo "About to download JS assets to ${VENDORS_PATH}" - -echo "getting video.js" -wget -c https://github.com/videojs/video.js/releases/download/v7.6.4/video-js-7.6.4.zip -rm -rf $VENDORS_PATH/videojs -mkdir -p $VENDORS_PATH/videojs -unzip -o -d $VENDORS_PATH/videojs video-js-7.6.4.zip -rm -rf $VENDORS_PATH/videojs/alt $VENDORS_PATH/videojs/examples -rm -f video-js-7.6.4.zip - -echo "getting ogv.js" -wget -c https://github.com/brion/ogv.js/releases/download/1.6.1/ogvjs-1.6.1.zip -rm -rf $VENDORS_PATH/ogvjs -unzip -o ogvjs-1.6.1.zip -mv ogvjs-1.6.1 $VENDORS_PATH/ogvjs -rm -f ogvjs-1.6.1.zip - -echo "getting videojs-ogvjs.js" -wget -c https://github.com/hartman/videojs-ogvjs/archive/v1.3.1.zip -rm -f $VENDORS_PATH/videojs-ogvjs.js -unzip -o v1.3.1.zip -mv videojs-ogvjs-1.3.1/dist/videojs-ogvjs.js $VENDORS_PATH/videojs-ogvjs.js -rm -rf videojs-ogvjs-1.3.1 -rm -f v1.3.1.zip - -if command -v fix_ogvjs_dist > /dev/null; then - echo "fixing JS files" - fix_ogvjs_dist $VENDORS_PATH "vendors" -else - echo "NOT fixing JS files (zimscraperlib not installed)" -fi - -echo "getting jquery.js" -wget -c -O $VENDORS_PATH/jquery.min.js https://code.jquery.com/jquery-3.4.1.min.js - -echo "getting bootstrap" -wget -c https://github.com/twbs/bootstrap/archive/v4.4.1.zip -rm -rf $VENDORS_PATH/bootstrap -unzip -o v4.4.1.zip -mv bootstrap-4.4.1/dist/ $VENDORS_PATH/bootstrap -rm -rf bootstrap-4.4.1 -rm -f v4.4.1.zip - -echo "getting pouchdb" -wget -c -O $VENDORS_PATH/pouchdb.min.js https://cdn.jsdelivr.net/npm/pouchdb@7.1.1/dist/pouchdb.min.js -wget -c -O $VENDORS_PATH/pouchdb.find.min.js https://cdn.jsdelivr.net/npm/pouchdb@7.1.1/dist/pouchdb.find.min.js - -echo "getting ScrollMagic" -wget -c -O $VENDORS_PATH/ScrollMagic.min.js https://cdnjs.cloudflare.com/ajax/libs/ScrollMagic/2.0.7/ScrollMagic.min.js - -echo "getting SugarJS" -wget -c -O $VENDORS_PATH/sugar.min.js https://raw.githubusercontent.com/andrewplummer/Sugar/2.0.4/dist/sugar.min.js - - -echo "getting vector icons" -wget -c https://github.com/dmhendricks/file-icon-vectors/releases/download/1.0/file-icon-vectors-1.0.zip -rm -rf $VENDORS_PATH/ext-icons -mkdir -p file-icon-vectors -unzip -o file-icon-vectors-1.0.zip -d file-icon-vectors -mv file-icon-vectors/dist/icons/square-o $VENDORS_PATH/ext-icons -cp -v $VENDORS_PATH/ext-icons/ppt.svg $VENDORS_PATH/ext-icons/odp.svg -rm -rf file-icon-vectors -rm -f file-icon-vectors-1.0.zip - -echo "getting handlebars.js" -wget -c -O $VENDORS_PATH/handlebars.runtime.min-v4.7.7.js https://s3.amazonaws.com/builds.handlebarsjs.com/handlebars.runtime.min-v4.7.7.js - diff --git a/openzim.toml b/openzim.toml new file mode 100644 index 0000000..15c02d4 --- /dev/null +++ b/openzim.toml @@ -0,0 +1,68 @@ +[files.assets.config] +target_dir="src/nautiluszim/templates/vendors" +execute_after=[ + "fix_ogvjs_dist .", + "cp -v ext-icons/ppt.svg ext-icons/odp.svg", + "handlebars .. -f ../precompiled.js", +] + +[files.assets.actions."video.js"] +action="extract_all" +source="https://github.com/videojs/video.js/releases/download/v8.12.0/video-js-8.12.0.zip" +target_dir="videojs" +remove = ["alt","examples",] + +[files.assets.actions."ogv.js"] +action="extract_items" +source="https://github.com/brion/ogv.js/releases/download/1.9.0/ogvjs-1.9.0.zip" +zip_paths=["ogvjs-1.9.0"] +target_paths=["ogvjs"] +remove = ["ogvjs/COPYING","ogvjs/*.txt","ogvjs/*.md",] + +[files.assets.actions."videojs-ogvjs.js"] +action="extract_items" +source="https://github.com/hartman/videojs-ogvjs/archive/v1.3.1.zip" +zip_paths=["videojs-ogvjs-1.3.1/dist/videojs-ogvjs.js"] +target_paths=["videojs-ogvjs.js"] + +[files.assets.actions."jquery.min.js"] +action="get_file" +source="https://code.jquery.com/jquery-3.5.1.min.js" +target_file="jquery.min.js" + +[files.assets.actions."bootstrap"] +action="extract_items" +source="https://github.com/twbs/bootstrap/archive/v4.4.1.zip" +zip_paths=["bootstrap-4.4.1/dist"] +target_paths=["bootstrap"] + +[files.assets.actions."pouchdb.min.js"] +action="get_file" +source="https://cdn.jsdelivr.net/npm/pouchdb@7.1.1/dist/pouchdb.min.js" +target_file="pouchdb.min.js" + +[files.assets.actions."pouchdb.find.min.js"] +action="get_file" +source="https://cdn.jsdelivr.net/npm/pouchdb@7.1.1/dist/pouchdb.find.min.js" +target_file="pouchdb.find.min.js" + +[files.assets.actions."ScrollMagic.min.js"] +action="get_file" +source="https://cdnjs.cloudflare.com/ajax/libs/ScrollMagic/2.0.7/ScrollMagic.min.js" +target_file="ScrollMagic.min.js" + +[files.assets.actions."sugar.min.js"] +action="get_file" +source="https://raw.githubusercontent.com/andrewplummer/Sugar/2.0.4/dist/sugar.min.js" +target_file="sugar.min.js" + +[files.assets.actions."vector-icon"] +action="extract_items" +source="https://github.com/dmhendricks/file-icon-vectors/releases/download/1.0/file-icon-vectors-1.0.zip" +zip_paths=["dist/icons/square-o"] +target_paths=["ext-icons"] + +[files.assets.actions."handlebars.js"] +action="get_file" +source="https://s3.amazonaws.com/builds.handlebarsjs.com/handlebars.runtime.min-v4.7.7.js" +target_file="handlebars.runtime.min-v4.7.7.js" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c4f74db --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,230 @@ +[build-system] +requires = ["hatchling", "hatch-openzim"] +build-backend = "hatchling.build" + +[project] +name = "nautiluszim" +requires-python = ">=3.11,<3.13" +description = "turns a collection of documents into a browsable ZIM file" +readme = "README.md" +dependencies = [ + "jinja2==3.1.4", + "zimscraperlib==3.3.2", +] +dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"] + +[project.optional-dependencies] +scripts = [ + "invoke==2.2.0", +] +lint = [ + "black==24.4.2", + "ruff==0.4.4", +] +check = [ + "pyright==1.1.362", +] +test = [ + "pytest==8.2.0", + "coverage==7.5.1", +] +dev = [ + "pre-commit==3.7.0", + "ipython==8.24.0", + "nautiluszim[scripts]", + "nautiluszim[lint]", + "nautiluszim[test]", + "nautiluszim[check]", +] + +[project.scripts] +nautiluszim = "nautiluszim.entrypoint:main" + +[tool.hatch.version] +path = "src/nautiluszim/__about__.py" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.metadata.hooks.openzim-metadata] +kind = "scraper" +additional-keywords = ["ted"] + +[tool.hatch.build.hooks.openzim-build] +dependencies = [ "zimscraperlib==3.3.2"] # required for fix_ogv_dist + +[tool.hatch.build] +exclude = [ + "/.github", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/nautiluszim"] + +[tool.hatch.envs.default] +features = ["dev"] + +[tool.hatch.envs.test] +features = ["scripts", "test"] + +[[tool.hatch.envs.test.matrix]] +python = ["3.11"] + +[tool.hatch.envs.test.scripts] +run = "inv test --args '{args}'" +run-cov = "inv test-cov --args '{args}'" +report-cov = "inv report-cov" +coverage = "inv coverage --args '{args}'" +html = "inv coverage --html --args '{args}'" + +[tool.hatch.envs.lint] +template = "lint" +skip-install = false +features = ["scripts", "lint"] + +[tool.hatch.envs.lint.scripts] +black = "inv lint-black --args '{args}'" +ruff = "inv lint-ruff --args '{args}'" +all = "inv lintall --args '{args}'" +fix-black = "inv fix-black --args '{args}'" +fix-ruff = "inv fix-ruff --args '{args}'" +fixall = "inv fixall --args '{args}'" + +[tool.hatch.envs.check] +features = ["scripts", "check"] + +[tool.hatch.envs.check.scripts] +pyright = "inv check-pyright --args '{args}'" +all = "inv checkall --args '{args}'" + +[tool.black] +line-length = 88 +target-version = ['py311'] + +[tool.ruff] +target-version = "py311" +line-length = 88 +src = ["src"] + +[tool.ruff.lint] +select = [ + "A", # flake8-builtins + # "ANN", # flake8-annotations + "ARG", # flake8-unused-arguments + # "ASYNC", # flake8-async + "B", # flake8-bugbear + # "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "C90", # mccabe + # "COM", # flake8-commas + # "D", # pydocstyle + # "DJ", # flake8-django + "DTZ", # flake8-datetimez + "E", # pycodestyle (default) + "EM", # flake8-errmsg + # "ERA", # eradicate + # "EXE", # flake8-executable + "F", # Pyflakes (default) + # "FA", # flake8-future-annotations + "FBT", # flake8-boolean-trap + # "FLY", # flynt + # "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + # "INP", # flake8-no-pep420 + # "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + # "NPY", # NumPy-specific rules + # "PD", # pandas-vet + # "PGH", # pygrep-hooks + # "PIE", # flake8-pie + # "PL", # Pylint + "PLC", # Pylint: Convention + "PLE", # Pylint: Error + "PLR", # Pylint: Refactor + "PLW", # Pylint: Warning + # "PT", # flake8-pytest-style + # "PTH", # flake8-use-pathlib + # "PYI", # flake8-pyi + "Q", # flake8-quotes + # "RET", # flake8-return + # "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "S", # flake8-bandit + # "SIM", # flake8-simplify + # "SLF", # flake8-self + "T10", # flake8-debugger + "T20", # flake8-print + # "TCH", # flake8-type-checking + # "TD", # flake8-todos + "TID", # flake8-tidy-imports + # "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + # Allow non-abstract empty methods in abstract base classes + "B027", + # Remove flake8-errmsg since we consider they bloat the code and provide limited value + "EM", + # Allow boolean positional values in function calls, like `dict.get(... True)` + "FBT003", + # Ignore checks for possible passwords + "S105", "S106", "S107", + # Ignore warnings on subprocess.run / popen + "S603", + # Ignore complexity + "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", + # allow datetime without timezone + "DTZ005", + # allow explicit string concatenation + "ISC003", +] +unfixable = [ + # Don't touch unused imports + "F401", +] + +[tool.ruff.lint.isort] +known-first-party = ["nautiluszim"] + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.lint.per-file-ignores] +# Tests can use magic values, assertions, and relative imports +"tests/**/*" = ["PLR2004", "S101", "TID252"] + +[tool.pytest.ini_options] +minversion = "7.3" +testpaths = ["tests"] +pythonpath = [".", "src"] + +[tool.coverage.paths] +nautiluszim = ["src/nautiluszim"] +tests = ["tests"] + +[tool.coverage.run] +source_pkgs = ["nautiluszim"] +branch = true +parallel = true +omit = [ + "src/nautiluszim/__about__.py", +] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.pyright] +include = ["src", "tests", "tasks.py"] +exclude = [".env/**", ".venv/**"] +extraPaths = ["src"] +pythonVersion = "3.11" +typeCheckingMode="basic" +disableBytesTypePromotions = true diff --git a/src/nautiluszim/__about__.py b/src/nautiluszim/__about__.py new file mode 100644 index 0000000..a82b376 --- /dev/null +++ b/src/nautiluszim/__about__.py @@ -0,0 +1 @@ +__version__ = "1.1.1" diff --git a/src/nautiluszim/__main__.py b/src/nautiluszim/__main__.py deleted file mode 100644 index abdf81c..0000000 --- a/src/nautiluszim/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from nautiluszim.entrypoint import main as entry - -if __name__ == "__main__": - entry() diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..a95c71a --- /dev/null +++ b/tasks.py @@ -0,0 +1,109 @@ +# pyright: strict, reportUntypedFunctionDecorator=false +import os + +from invoke.context import Context +from invoke.tasks import task # pyright: ignore [reportUnknownVariableType] + +use_pty = not os.getenv("CI", "") + + +@task(optional=["args"], help={"args": "pytest additional arguments"}) +def test(ctx: Context, args: str = ""): + """run tests (without coverage)""" + ctx.run(f"pytest {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "pytest additional arguments"}) +def test_cov(ctx: Context, args: str = ""): + """run test vith coverage""" + ctx.run(f"coverage run -m pytest {args}", pty=use_pty) + + +@task(optional=["html"], help={"html": "flag to export html report"}) +def report_cov(ctx: Context, *, html: bool = False): + """report coverage""" + ctx.run("coverage combine", warn=True, pty=use_pty) + ctx.run("coverage report --show-missing", pty=use_pty) + if html: + ctx.run("coverage html", pty=use_pty) + + +@task( + optional=["args", "html"], + help={ + "args": "pytest additional arguments", + "html": "flag to export html report", + }, +) +def coverage(ctx: Context, args: str = "", *, html: bool = False): + """run tests and report coverage""" + test_cov(ctx, args=args) + report_cov(ctx, html=html) + + +@task(optional=["args"], help={"args": "black additional arguments"}) +def lint_black(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("black --version", pty=use_pty) + ctx.run(f"black --check --diff {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def lint_ruff(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff check {args}", pty=use_pty) + + +@task( + optional=["args"], + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, +) +def lintall(ctx: Context, args: str = "."): + """Check linting""" + args = args or "." # needed for hatch script + lint_black(ctx, args) + lint_ruff(ctx, args) + + +@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) +def check_pyright(ctx: Context, args: str = ""): + """check static types with pyright""" + ctx.run("pyright --version") + ctx.run(f"pyright {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) +def checkall(ctx: Context, args: str = ""): + """check static types""" + check_pyright(ctx, args) + + +@task(optional=["args"], help={"args": "black additional arguments"}) +def fix_black(ctx: Context, args: str = "."): + """fix black formatting""" + args = args or "." # needed for hatch script + ctx.run(f"black {args}", pty=use_pty) + + +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def fix_ruff(ctx: Context, args: str = "."): + """fix all ruff rules""" + args = args or "." # needed for hatch script + ctx.run(f"ruff check --fix {args}", pty=use_pty) + + +@task( + optional=["args"], + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, +) +def fixall(ctx: Context, args: str = "."): + """Fix everything automatically""" + args = args or "." # needed for hatch script + fix_black(ctx, args) + fix_ruff(ctx, args) + lintall(ctx, args) diff --git a/tests/test_stub.py b/tests/test_stub.py new file mode 100644 index 0000000..54e4e5b --- /dev/null +++ b/tests/test_stub.py @@ -0,0 +1,5 @@ +from nautiluszim.__about__ import __version__ + + +def test_version(): + assert __version__[0].isdigit()