From c39b4fda40c88737f1c56f5ad6f42cbed974478b Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Mon, 23 Oct 2023 07:41:31 +0200 Subject: [PATCH] Dependencies: Add support for Python 3.12 The file `requirements/requirements-py-3.12.txt` is added which provides a complete environment for Python 3.12. The CI is updated to add Python 3.12 in all strategy matrices or replace Python 3.11 where only the oldest and latest Python version are tested. Note that the Python version for the `pre-commit` jobs are kept at 3.10 for now. The reason is that in Python 3.12 f-strings are improved by allowing nested quotes. For example: f'some_dict['key']' is now supported, whereas before Python 3.12 this would not work since the nested quotes would not be parsed correctly and the internal quotes had to be either escaped or changed for double quotes. A number of dependencies had to be updated to make them compatible with Python 3.12, usually because older version still relied on the `distutils` and `pkg_resources` standard lib modules which have been removed. The `utils/dependency_management.py` had to be updated similarly to replace `pkg_resources` with `packaging`. The latter had to be updated to `packaging==23.0` in order to have the `__eq__` implementation for the `Requirement` class which the script relies on. The memory leak tests are skipped on Python 3.12 because currently they hang. The problem is with the `pympler.muppy.get_objects` method. This calls `gc.collect` internally, but that call is blocking. The exact cause is as of yet unknown. The garbage collecting has been changed in Python 3.12 so it is not completely unexpected either. The `sphinxcontrib-details-directive` dependency is removed. It was used for the sphinx extension to add the ports of port namespaces in HTML's `
` tags, allowing them to be collapsed. This could help with readability in case of large namespaces. However, this package breaks on Python 3.12 since it imports the deprecated `pkg_resources` package. Since the package has not been maintained since 4 years, it is unlikely this will be fixed it and so instead it is removed for now. See https://github.com/sphinx-contrib/sphinxcontrib-details-directive --- .../system_tests/pytest/test_memory_leaks.py | 7 + .github/workflows/ci-code.yml | 8 +- .github/workflows/test-install.yml | 6 +- aiida/sphinxext/__init__.py | 1 - aiida/sphinxext/process.py | 8 +- pyproject.toml | 20 +- requirements/requirements-py-3.10.txt | 24 +- requirements/requirements-py-3.11.txt | 24 +- requirements/requirements-py-3.12.txt | 211 ++++++++++++++++++ requirements/requirements-py-3.9.txt | 24 +- tests/sphinxext/conftest.py | 3 +- tests/sphinxext/sources/workchain/conf.py | 2 +- .../sources/workchain_broken/conf.py | 2 +- .../test_workchain/test_workchain_build.xml | 42 ++-- utils/dependency_management.py | 71 +++--- utils/requirements.txt | 4 +- 16 files changed, 335 insertions(+), 122 deletions(-) create mode 100644 requirements/requirements-py-3.12.txt diff --git a/.github/system_tests/pytest/test_memory_leaks.py b/.github/system_tests/pytest/test_memory_leaks.py index 396793346f..cdd7a536ab 100644 --- a/.github/system_tests/pytest/test_memory_leaks.py +++ b/.github/system_tests/pytest/test_memory_leaks.py @@ -8,6 +8,10 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Utilities for testing memory leakage.""" +import sys + +import pytest + from aiida import orm from aiida.engine import processes, run_get_node from aiida.plugins import CalculationFactory @@ -23,6 +27,7 @@ def run_finished_ok(*args, **kwargs): assert node.is_finished_ok, (node.exit_status, node.exit_message) +@pytest.mark.skipif(sys.version_info >= (3, 12), reason='Garbage collecting hangs on Python 3.12') def test_leak_run_process(): """Test whether running a dummy process leaks memory.""" inputs = {'a': orm.Int(2), 'b': orm.Str('test')} @@ -34,6 +39,7 @@ def test_leak_run_process(): assert not process_instances, f'Memory leak: process instances remain in memory: {process_instances}' +@pytest.mark.skipif(sys.version_info >= (3, 12), reason='Garbage collecting hangs on Python 3.12') def test_leak_local_calcjob(aiida_local_code_factory): """Test whether running a local CalcJob leaks memory.""" inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': aiida_local_code_factory('core.arithmetic.add', '/bin/bash')} @@ -45,6 +51,7 @@ def test_leak_local_calcjob(aiida_local_code_factory): assert not process_instances, f'Memory leak: process instances remain in memory: {process_instances}' +@pytest.mark.skipif(sys.version_info >= (3, 12), reason='Garbage collecting hangs on Python 3.12') def test_leak_ssh_calcjob(): """Test whether running a CalcJob over SSH leaks memory. diff --git a/.github/workflows/ci-code.yml b/.github/workflows/ci-code.yml index 2d91760f67..cb77c2f137 100644 --- a/.github/workflows/ci-code.yml +++ b/.github/workflows/ci-code.yml @@ -23,10 +23,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v2 with: - python-version: '3.9' + python-version: '3.12' - name: Install utils/ dependencies run: pip install -r utils/requirements.txt @@ -55,7 +55,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.11'] + python-version: ['3.9', '3.12'] services: postgres: @@ -131,7 +131,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.11'] + python-version: ['3.9', '3.12'] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index d25c6b9c24..e3d17ae742 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -58,7 +58,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v2 @@ -144,7 +144,7 @@ jobs: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] # Not being able to install with conda on a specific Python version is # not sufficient to fail the run, but something we want to be aware of. @@ -195,7 +195,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] services: postgres: diff --git a/aiida/sphinxext/__init__.py b/aiida/sphinxext/__init__.py index 6ea2aa4cc9..8052d877c4 100644 --- a/aiida/sphinxext/__init__.py +++ b/aiida/sphinxext/__init__.py @@ -16,7 +16,6 @@ def setup(app): from . import calcjob, process, workchain - app.setup_extension('sphinxcontrib.details.directive') process.setup_extension(app) workchain.setup_extension(app) calcjob.setup_extension(app) diff --git a/aiida/sphinxext/process.py b/aiida/sphinxext/process.py index 759617f5e2..5610ab1fb9 100644 --- a/aiida/sphinxext/process.py +++ b/aiida/sphinxext/process.py @@ -20,7 +20,6 @@ from sphinx import addnodes from sphinx.ext.autodoc import ClassDocumenter from sphinx.util.docutils import SphinxDirective -from sphinxcontrib.details.directive import details, summary from aiida.common.utils import get_object_from_string from aiida.engine import Process @@ -156,12 +155,7 @@ def build_portnamespace_doctree(self, port_namespace): if port.help is not None: item += nodes.Text(' -- ') item.extend(publish_doctree(port.help)[0].children) - sub_doctree = self.build_portnamespace_doctree(port) - if sub_doctree: - sub_item = details(opened=self.EXPAND_NAMESPACES_FLAG in self.options) - sub_item += summary(text='Namespace Ports') - sub_item += sub_doctree - item += sub_item + item += self.build_portnamespace_doctree(port) else: raise NotImplementedError result += item diff --git a/pyproject.toml b/pyproject.toml index 0eaa9195b7..07823b57b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering" ] keywords = ["aiida", "workflows"] @@ -78,15 +79,14 @@ rest = [ ] docs = [ "pydata-sphinx-theme~=0.13.3", - "sphinx~=4.1", - "sphinxcontrib-details-directive~=0.1.0", + "sphinx~=7.2", "sphinx-copybutton~=0.5.0", - "sphinx-design~=0.0.13", - "sphinx-notfound-page~=0.5", + "sphinx-design~=0.5.0", + "sphinx-notfound-page~=1.0", "sphinxext-rediraffe~=0.2.4", # "sphinx-sqlalchemy~=0.1.1", "sphinx-intl~=2.1.0", - "myst-nb~=0.17.0", + "myst-nb~=1.0.0", ] atomic_tools = [ "PyCifRW~=4.4", @@ -104,7 +104,7 @@ notebook = [ ] pre-commit = [ "mypy==0.991", - "packaging~=20.9", + "packaging~=23.0", "pre-commit~=2.2", "pylint~=2.17.4", "pylint-aiida~=0.1.1", @@ -118,15 +118,15 @@ tests = [ "pgtest~=1.3,>=1.3.1", "pytest~=7.0", "pytest-asyncio~=0.12,<0.17", - "pytest-timeout~=1.3", + "pytest-timeout~=2.0", "pytest-cov~=2.7,<2.11", - "pytest-rerunfailures~=9.1,>=9.1.1", + "pytest-rerunfailures~=12.0", "pytest-benchmark~=4.0", "pytest-regressions~=2.2", "pympler~=0.9", "coverage~=6.0", - "sphinx~=4.0", - "docutils==0.16", + "sphinx~=7.2", + "docutils~=0.20", ] bpython = [ "bpython~=0.18.0" diff --git a/requirements/requirements-py-3.10.txt b/requirements/requirements-py-3.10.txt index e7231a59c3..181c516203 100644 --- a/requirements/requirements-py-3.10.txt +++ b/requirements/requirements-py-3.10.txt @@ -43,7 +43,8 @@ defusedxml==0.7.1 deprecation==2.1.0 disk-objectstore==1.0.0 docstring-parser==0.15 -docutils==0.16 +docutils==0.20.1 +emmet-core==0.57.1 exceptiongroup==1.1.1 executing==1.2.0 fastjsonschema==2.17.1 @@ -81,19 +82,19 @@ kiwisolver==1.4.4 latexcodec==2.0.1 linkify-it-py==2.0.2 mako==1.2.4 -markdown-it-py[linkify,plugins]==2.2.0 +markdown-it-py[linkify,plugins]==3.0.0 markupsafe==2.1.3 matplotlib==3.7.1 matplotlib-inline==0.1.6 -mdit-py-plugins==0.3.5 +mdit-py-plugins==0.4.0 mdurl==0.1.2 mistune==3.0.1 monty==2023.9.25 mpmath==1.3.0 msgpack==1.0.5 multidict==6.0.4 -myst-nb==0.17.2 -myst-parser==0.18.1 +myst-nb==1.0.0 +myst-parser==2.0.0 nbclassic==1.0.0 nbclient==0.7.4 nbconvert==7.6.0 @@ -146,8 +147,8 @@ pytest-benchmark==4.0.0 pytest-cov==2.10.1 pytest-datadir==1.4.1 pytest-regressions==2.4.2 -pytest-rerunfailures==9.1.1 -pytest-timeout==1.4.2 +pytest-rerunfailures==12.0.0 +pytest-timeout==2.2.0 python-dateutil==2.8.2 python-json-logger==2.0.7 python-memcached==1.59 @@ -173,18 +174,17 @@ sniffio==1.3.0 snowballstemmer==2.2.0 soupsieve==2.4.1 spglib==2.0.2 -sphinx==4.5.0 +sphinx==7.2.6 sphinx-copybutton==0.5.2 -sphinx-design==0.0.13 +sphinx-design==0.5.0 sphinx-intl==2.1.0 -sphinx-notfound-page==0.8.3 +sphinx-notfound-page==1.0.0 sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-details-directive==0.1.0 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 sphinxext-rediraffe==0.2.7 sqlalchemy==2.0.23 stack-data==0.6.2 diff --git a/requirements/requirements-py-3.11.txt b/requirements/requirements-py-3.11.txt index bd0fe3749e..b373c77c32 100644 --- a/requirements/requirements-py-3.11.txt +++ b/requirements/requirements-py-3.11.txt @@ -43,7 +43,8 @@ defusedxml==0.7.1 deprecation==2.1.0 disk-objectstore==1.0.0 docstring-parser==0.15 -docutils==0.16 +docutils==0.20.1 +emmet-core==0.57.1 executing==1.2.0 fastjsonschema==2.17.1 flask==2.3.2 @@ -80,19 +81,19 @@ kiwisolver==1.4.4 latexcodec==2.0.1 linkify-it-py==2.0.2 mako==1.2.4 -markdown-it-py[linkify,plugins]==2.2.0 +markdown-it-py[linkify,plugins]==3.0.0 markupsafe==2.1.3 matplotlib==3.7.1 matplotlib-inline==0.1.6 -mdit-py-plugins==0.3.5 +mdit-py-plugins==0.4.0 mdurl==0.1.2 mistune==3.0.1 monty==2023.9.25 mpmath==1.3.0 msgpack==1.0.5 multidict==6.0.4 -myst-nb==0.17.2 -myst-parser==0.18.1 +myst-nb==1.0.0 +myst-parser==2.0.0 nbclassic==1.0.0 nbclient==0.7.4 nbconvert==7.6.0 @@ -145,8 +146,8 @@ pytest-benchmark==4.0.0 pytest-cov==2.10.1 pytest-datadir==1.4.1 pytest-regressions==2.4.2 -pytest-rerunfailures==9.1.1 -pytest-timeout==1.4.2 +pytest-rerunfailures==12.0.0 +pytest-timeout==2.2.0 python-dateutil==2.8.2 python-json-logger==2.0.7 python-memcached==1.59 @@ -172,18 +173,17 @@ sniffio==1.3.0 snowballstemmer==2.2.0 soupsieve==2.4.1 spglib==2.0.2 -sphinx==4.5.0 +sphinx==7.2.6 sphinx-copybutton==0.5.2 -sphinx-design==0.0.13 +sphinx-design==0.5.0 sphinx-intl==2.1.0 -sphinx-notfound-page==0.8.3 +sphinx-notfound-page==1.0.0 sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-details-directive==0.1.0 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 sphinxext-rediraffe==0.2.7 sqlalchemy==2.0.23 stack-data==0.6.2 diff --git a/requirements/requirements-py-3.12.txt b/requirements/requirements-py-3.12.txt new file mode 100644 index 0000000000..9b4cc0a633 --- /dev/null +++ b/requirements/requirements-py-3.12.txt @@ -0,0 +1,211 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --extra=atomic_tools --extra=docs --extra=notebook --extra=rest --extra=tests --no-annotate --output-file=requirements/requirements-py-3.12.txt pyproject.toml +# +accessible-pygments==0.0.4 +aiida-export-migration-tests==0.9.0 +aio-pika==6.8.1 +aiormq==3.3.1 +alabaster==0.7.13 +alembic==1.12.0 +aniso8601==9.0.1 +anyio==4.0.0 +archive-path==0.4.2 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +ase==3.22.1 +asn1crypto==1.5.1 +asttokens==2.4.0 +async-generator==1.10 +attrs==23.1.0 +babel==2.13.1 +backcall==0.2.0 +bcrypt==4.0.1 +beautifulsoup4==4.12.2 +bleach==6.1.0 +blinker==1.6.3 +certifi==2023.7.22 +cffi==1.16.0 +charset-normalizer==3.3.1 +circus==0.18.0 +click==8.1.7 +click-spinner==0.1.10 +comm==0.1.4 +contourpy==1.1.1 +coverage==6.5.0 +cryptography==41.0.5 +cycler==0.12.1 +debugpy==1.8.0 +decorator==5.1.1 +defusedxml==0.7.1 +deprecation==2.1.0 +disk-objectstore==1.0.0 +docstring-parser==0.15 +docutils==0.20.1 +executing==2.0.0 +fastjsonschema==2.18.1 +flask==2.3.3 +flask-cors==3.0.10 +flask-restful==0.3.10 +fonttools==4.43.1 +future==0.18.3 +graphviz==0.20.1 +greenlet==3.0.0 +idna==3.4 +imagesize==1.4.1 +importlib-metadata==6.8.0 +iniconfig==2.0.0 +ipykernel==6.25.2 +ipython==8.16.1 +ipython-genutils==0.2.0 +ipywidgets==8.1.1 +itsdangerous==2.1.2 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema[format-nongpl]==3.2.0 +jupyter==1.0.0 +jupyter-cache==0.6.1 +jupyter-client==8.4.0 +jupyter-console==6.6.3 +jupyter-core==5.4.0 +jupyter-events==0.6.3 +jupyter-server==2.8.0 +jupyter-server-terminals==0.4.4 +jupyterlab-pygments==0.2.2 +jupyterlab-widgets==3.0.9 +kiwipy[rmq]==0.7.8 +kiwisolver==1.4.5 +latexcodec==2.0.1 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.3 +matplotlib==3.8.0 +matplotlib-inline==0.1.6 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mistune==3.0.2 +monty==2023.9.25 +mpmath==1.3.0 +multidict==6.0.4 +myst-nb==1.0.0 +myst-parser==2.0.0 +nbclassic==1.0.0 +nbclient==0.7.4 +nbconvert==7.9.2 +nbformat==5.9.2 +nest-asyncio==1.5.8 +networkx==3.2 +notebook==6.5.4 +notebook-shim==0.2.3 +numpy==1.26.1 +overrides==7.4.0 +packaging==23.2 +palettable==3.3.3 +pamqp==2.3.0 +pandas==2.1.1 +pandocfilters==1.5.0 +paramiko==2.12.0 +parso==0.8.3 +pexpect==4.8.0 +pg8000==1.30.2 +pgsu==0.2.4 +pgtest==1.3.2 +pickleshare==0.7.5 +pillow==10.1.0 +platformdirs==3.11.0 +plotly==5.17.0 +pluggy==1.3.0 +plumpy==0.21.10 +prometheus-client==0.17.1 +prompt-toolkit==3.0.39 +psutil==5.9.6 +psycopg2-binary==2.9.9 +ptyprocess==0.7.0 +pure-eval==0.2.2 +py-cpuinfo==9.0.0 +pybtex==0.24.0 +pycifrw==4.4.5 +pycparser==2.21 +pydantic==2.4.0 +pydata-sphinx-theme==0.13.3 +pygments==2.16.1 +pymatgen==2023.10.11 +pympler==0.9 +pymysql==0.9.3 +pynacl==1.5.0 +pyparsing==2.4.7 +pyrsistent==0.19.3 +pytest==7.4.2 +pytest-asyncio==0.16.0 +pytest-benchmark==4.0.0 +pytest-cov==2.10.1 +pytest-datadir==1.5.0 +pytest-regressions==2.5.0 +pytest-rerunfailures==12.0.0 +pytest-timeout==2.2.0 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +python-memcached==1.59 +pytray==0.3.4 +pytz==2021.3 +pyyaml==6.0.1 +pyzmq==25.1.1 +qtconsole==5.4.4 +qtpy==2.4.1 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +ruamel-yaml==0.18.2 +ruamel-yaml-clib==0.2.8 +scipy==1.11.3 +scramp==1.4.4 +seekpath==1.9.7 +send2trash==1.8.2 +shortuuid==1.0.11 +six==1.16.0 +sniffio==1.3.0 +snowballstemmer==2.2.0 +soupsieve==2.5 +spglib==2.1.0 +sphinx==7.2.6 +sphinx-copybutton==0.5.2 +sphinx-design==0.5.0 +sphinx-intl==2.1.0 +sphinx-notfound-page==1.0.0 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.9 +sphinxext-rediraffe==0.2.7 +sqlalchemy==2.0.23 +sqlalchemy-utils==0.37.9 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.8.10 +tenacity==8.2.3 +terminado==0.17.1 +tinycss2==1.2.1 +tornado==6.3.3 +tqdm==4.66.1 +traitlets==5.11.2 +typing-extensions==4.8.0 +tzdata==2023.3 +uncertainties==3.1.7 +upf-to-json==0.9.5 +urllib3==2.0.7 +wcwidth==0.2.8 +webencodings==0.5.1 +websocket-client==1.6.4 +werkzeug==3.0.0 +widgetsnbextension==4.0.9 +wrapt==1.15.0 +yarl==1.9.2 +zipp==3.17.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/requirements-py-3.9.txt b/requirements/requirements-py-3.9.txt index 123b833a5e..552b0a5248 100644 --- a/requirements/requirements-py-3.9.txt +++ b/requirements/requirements-py-3.9.txt @@ -43,7 +43,8 @@ defusedxml==0.7.1 deprecation==2.1.0 disk-objectstore==1.0.0 docstring-parser==0.15 -docutils==0.16 +docutils==0.20.1 +emmet-core==0.57.1 exceptiongroup==1.1.1 executing==1.2.0 fastjsonschema==2.17.1 @@ -83,19 +84,19 @@ kiwisolver==1.4.4 latexcodec==2.0.1 linkify-it-py==2.0.2 mako==1.2.4 -markdown-it-py[linkify,plugins]==2.2.0 +markdown-it-py[linkify,plugins]==3.0.0 markupsafe==2.1.3 matplotlib==3.7.1 matplotlib-inline==0.1.6 -mdit-py-plugins==0.3.5 +mdit-py-plugins==0.4.0 mdurl==0.1.2 mistune==3.0.1 monty==2023.9.25 mpmath==1.3.0 msgpack==1.0.5 multidict==6.0.4 -myst-nb==0.17.2 -myst-parser==0.18.1 +myst-nb==1.0.0 +myst-parser==2.0.0 nbclassic==1.0.0 nbclient==0.7.4 nbconvert==7.6.0 @@ -148,8 +149,8 @@ pytest-benchmark==4.0.0 pytest-cov==2.10.1 pytest-datadir==1.4.1 pytest-regressions==2.4.2 -pytest-rerunfailures==9.1.1 -pytest-timeout==1.4.2 +pytest-rerunfailures==12.0.0 +pytest-timeout==2.2.0 python-dateutil==2.8.2 python-json-logger==2.0.7 python-memcached==1.59 @@ -175,18 +176,17 @@ sniffio==1.3.0 snowballstemmer==2.2.0 soupsieve==2.4.1 spglib==2.0.2 -sphinx==4.5.0 +sphinx==7.2.6 sphinx-copybutton==0.5.2 -sphinx-design==0.0.13 +sphinx-design==0.5.0 sphinx-intl==2.1.0 -sphinx-notfound-page==0.8.3 +sphinx-notfound-page==1.0.0 sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-details-directive==0.1.0 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 sphinxext-rediraffe==0.2.7 sqlalchemy==2.0.23 stack-data==0.6.2 diff --git a/tests/sphinxext/conftest.py b/tests/sphinxext/conftest.py index da2ad97613..1cdf188a57 100644 --- a/tests/sphinxext/conftest.py +++ b/tests/sphinxext/conftest.py @@ -13,7 +13,6 @@ import sys import pytest -from sphinx.testing.path import path as sphinx_path from sphinx.testing.util import SphinxTestApp SRC_DIR = pathlib.Path(__file__).parent / 'sources' @@ -69,7 +68,7 @@ def _func(src_folder, **kwargs): filepath_source = SRC_DIR / src_folder filepath_target = tmp_path / src_folder shutil.copytree(filepath_source, filepath_target) - app = make_app(srcdir=sphinx_path(filepath_target.absolute()), **kwargs) + app = make_app(srcdir=filepath_target.absolute(), **kwargs) return SphinxBuild(app, filepath_target) yield _func diff --git a/tests/sphinxext/sources/workchain/conf.py b/tests/sphinxext/sources/workchain/conf.py index 14787e0625..2a6132749a 100644 --- a/tests/sphinxext/sources/workchain/conf.py +++ b/tests/sphinxext/sources/workchain/conf.py @@ -68,7 +68,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/tests/sphinxext/sources/workchain_broken/conf.py b/tests/sphinxext/sources/workchain_broken/conf.py index 14787e0625..2a6132749a 100644 --- a/tests/sphinxext/sources/workchain_broken/conf.py +++ b/tests/sphinxext/sources/workchain_broken/conf.py @@ -68,7 +68,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/tests/sphinxext/test_workchain/test_workchain_build.xml b/tests/sphinxext/test_workchain/test_workchain_build.xml index 3cf00c81f7..4ae9e6fa32 100644 --- a/tests/sphinxext/test_workchain/test_workchain_build.xml +++ b/tests/sphinxext/test_workchain/test_workchain_build.xml @@ -1,6 +1,6 @@ - +
sphinx-aiida demo @@ -8,12 +8,12 @@ - workchaindemo_workchain.DemoWorkChain + workchaindemo_workchain.DemoWorkChain A demo workchain to show how the workchain auto-documentation works. - Inputs:metadata, Namespace
Namespace Portscall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.
nsp, Namespace – A separate namespace, nsp.nsp2, Namespacex, Float, required – First input argument.y, Namespace
Namespace Portsnested, Namespace – A nested namespace.
Namespace Portsa, Int, required – An input in the nested namespace.
z, Int, required – Input in a separate namespace.
 + Inputs:metadata, Namespacecall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.nsp, Namespace – A separate namespace, nsp.nsp2, Namespacex, Float, required – First input argument.y, Namespacenested, Namespace – A nested namespace.a, Int, required – An input in the nested namespace.z, Int, required – Input in a separate namespace. Outputs:z, Bool, required – Output of the demoworkchain. Outline:start while(some_check) @@ -25,12 +25,12 @@ finalize If you want to hide the inputs that are not stored as nodes in the database, use the :hide-unstored-inputs: option. - workchaindemo_workchain.DemoWorkChain + workchaindemo_workchain.DemoWorkChain A demo workchain to show how the workchain auto-documentation works. - Inputs:metadata, Namespace
Namespace Portscall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.
nsp, Namespace – A separate namespace, nsp.nsp2, Namespacex, Float, required – First input argument.y, Namespace
Namespace Portsnested, Namespace – A nested namespace.
Namespace Portsa, Int, required – An input in the nested namespace.
z, Int, required – Input in a separate namespace.
 + Inputs:metadata, Namespacecall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.nsp, Namespace – A separate namespace, nsp.nsp2, Namespacex, Float, required – First input argument.y, Namespacenested, Namespace – A nested namespace.a, Int, required – An input in the nested namespace.z, Int, required – Input in a separate namespace. Outputs:z, Bool, required – Output of the demoworkchain. Outline:start while(some_check) @@ -42,12 +42,12 @@ finalize The namespaces can be set to expand by default, using the :expand-namespaces: option. - workchaindemo_workchain.DemoWorkChain + workchaindemo_workchain.DemoWorkChain A demo workchain to show how the workchain auto-documentation works. - Inputs:metadata, Namespace
Namespace Portscall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.
nsp, Namespace – A separate namespace, nsp.nsp2, Namespacex, Float, required – First input argument.y, Namespace
Namespace Portsnested, Namespace – A nested namespace.
Namespace Portsa, Int, required – An input in the nested namespace.
z, Int, required – Input in a separate namespace.
 + Inputs:metadata, Namespacecall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.nsp, Namespace – A separate namespace, nsp.nsp2, Namespacex, Float, required – First input argument.y, Namespacenested, Namespace – A nested namespace.a, Int, required – An input in the nested namespace.z, Int, required – Input in a separate namespace. Outputs:z, Bool, required – Output of the demoworkchain. Outline:start while(some_check) @@ -59,27 +59,27 @@ finalize The following workchain checks that the directive works also when no outline is specified: - workchaindemo_workchain.EmptyOutlineWorkChain + workchaindemo_workchain.EmptyOutlineWorkChain Here we check that the directive works even if the outline is empty. - Inputs:metadata, Namespace
Namespace Portscall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.
x, Float, required – First input argument. + Inputs:metadata, Namespacecall_link_label, str, optional, is_metadata – The label to use for the CALL link if the process is called by another process.description, (str, NoneType), optional, is_metadata – Description to set on the process node.label, (str, NoneType), optional, is_metadata – Label to set on the process node.store_provenance, bool, optional, is_metadata – If set to False provenance will not be stored in the database.x, Float, required – First input argument. Outputs:None defined. The command is also hooked into sphinx.ext.autodoc, so AiiDA processes will be properly documented using .. automodule:: as well. - - This module defines an example workchain for the aiida-workchain documentation directive. + + This module defines an example workchain for the aiida-workchain documentation directive. - - class demo_workchain.DemoWorkChain*args: Any**kwargs: Any + + class demo_workchain.DemoWorkChain*args: Any**kwargs: Any A demo workchain to show how the workchain auto-documentation works. - - classmethod definespec + + classmethod definespec Define the specification of the process, including its inputs, outputs and known exit codes. A metadata input namespace is defined, with optional ports that are not stored in the database. @@ -88,13 +88,13 @@ finalize - - class demo_workchain.EmptyOutlineWorkChain*args: Any**kwargs: Any + + class demo_workchain.EmptyOutlineWorkChain*args: Any**kwargs: Any Here we check that the directive works even if the outline is empty. - - classmethod definespec + + classmethod definespec Define the specification of the process, including its inputs, outputs and known exit codes. A metadata input namespace is defined, with optional ports that are not stored in the database. @@ -103,8 +103,8 @@ finalize - - class demo_workchain.NormalClass + + class demo_workchain.NormalClass This is here to check that we didn’t break the regular autoclass. diff --git a/utils/dependency_management.py b/utils/dependency_management.py index cb2b6dbb73..78a92c811e 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -17,9 +17,10 @@ import sys import click +from packaging.requirements import Requirement +from packaging.specifiers import Specifier from packaging.utils import canonicalize_name from packaging.version import parse -from pkg_resources import Requirement, parse_requirements import requests import tomli import yaml @@ -71,14 +72,14 @@ def _setuptools_to_conda(req): for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): if re.match(pattern, str(req)): - req = Requirement.parse(re.sub(pattern, replacement, str(req))) + req = Requirement(re.sub(pattern, replacement, str(req))) break # markers are not supported by conda req.marker = None # We need to parse the modified required again, to ensure consistency. - return Requirement.parse(str(req)) + return Requirement(str(req)) def _find_linenos_of_requirements_in_pyproject(requirements): @@ -101,22 +102,14 @@ def _find_linenos_of_requirements_in_pyproject(requirements): return linenos -class _Entry: - """Helper class to check whether a given distribution fulfills a requirement.""" - - def __init__(self, requirement): - self._req = requirement - - def fulfills(self, requirement): - """Returns True if this entry fullfills the requirement.""" - - return canonicalize_name(self._req.name) == canonicalize_name(requirement.name) \ - and self._req.specs[0][1] in requirement.specifier - - -def _parse_working_set(entries): - for req in parse_requirements(entries): - yield _Entry(req) +def parse_requirements(requirements): + """Parse requirements from a file or list of strings.""" + results = [] + for requirement in requirements: + stripped = requirement.strip() + if stripped and not stripped.startswith('#'): + results.append(Requirement(stripped)) + return results @click.group() @@ -137,7 +130,7 @@ def generate_environment_yml(): # Read the requirements from 'pyproject.toml' pyproject = _load_pyproject() - install_requirements = [Requirement.parse(r) for r in pyproject['project']['dependencies']] + install_requirements = [Requirement(r) for r in pyproject['project']['dependencies']] # python version cannot be overriden from outside environment.yml # (even if it is not specified at all in environment.yml) @@ -174,8 +167,8 @@ def validate_environment_yml(): # pylint: disable=too-many-branches # Read the requirements from 'pyproject.toml' and 'environment.yml'. pyproject = _load_pyproject() - install_requirements = [Requirement.parse(r) for r in pyproject['project']['dependencies']] - python_requires = Requirement.parse('python' + pyproject['project']['requires-python']) + install_requirements = [Requirement(r) for r in pyproject['project']['dependencies']] + python_requires = Requirement('python' + pyproject['project']['requires-python']) environment_yml = _load_environment_yml() try: @@ -187,7 +180,7 @@ def validate_environment_yml(): # pylint: disable=too-many-branches raise DependencySpecificationError(f"Error in 'environment.yml': {error}") try: - conda_dependencies = {Requirement.parse(d) for d in environment_yml['dependencies']} + conda_dependencies = {Requirement(d) for d in environment_yml['dependencies']} except TypeError as error: raise DependencySpecificationError(f"Error while parsing requirements from 'environment_yml': {error}") @@ -267,7 +260,7 @@ def validate_all(ctx): 'as part of a GitHub actions workflow. Note: Requires environment ' 'variable GITHUB_ACTIONS=true .' ) -def check_requirements(extras, github_annotate): # pylint disable: too-many-locals-too-many-branches +def check_requirements(extras, github_annotate): # pylint: disable=too-many-locals,too-many-branches """Check the 'requirements/*.txt' files. Checks that the environments specified in the requirements files @@ -294,13 +287,23 @@ def check_requirements(extras, github_annotate): # pylint disable: too-many-loc if not match: continue env = {'python_version': match.groups()[0]} - required = {r for r in install_requires if r.marker is None or r.marker.evaluate(env)} + requirements_abstract = {r for r in install_requires if r.marker is None or r.marker.evaluate(env)} + installed = [] with open(fn_req, encoding='utf8') as req_file: - working_set = list(_parse_working_set(req_file)) - installed = {req for req in required for entry in working_set if entry.fulfills(req)} - - for dependency in required.difference(installed): + requirements_concrete = parse_requirements(req_file) + + for requirement_abstract in requirements_abstract: + for requirement_concrete in requirements_concrete: + version = Specifier(str(requirement_concrete.specifier)).version + if ( + canonicalize_name(requirement_abstract.name) == canonicalize_name(requirement_concrete.name) and + requirement_abstract.specifier.contains(version) + ): + installed.append(requirement_abstract) + break + + for dependency in requirements_abstract.difference(set(installed)): not_installed[dependency].append(fn_req) if any(not_installed.values()): @@ -352,9 +355,9 @@ def show_requirements(extras, fmt): if 'all' in extras: extras = list(pyproject['project']['optional-dependencies']) - to_install = {Requirement.parse(r) for r in pyproject['project']['dependencies']} + to_install = {Requirement(r) for r in pyproject['project']['dependencies']} for key in extras: - to_install.update(Requirement.parse(r) for r in pyproject['project']['optional-dependencies'][key]) + to_install.update(Requirement(r) for r in pyproject['project']['optional-dependencies'][key]) if fmt == 'pip': click.echo('\n'.join(sorted(map(str, to_install)))) @@ -381,7 +384,7 @@ def pip_install_extras(extras): to_install = set() for key in extras: - to_install.update(Requirement.parse(r) for r in pyproject['project']['optional-dependencies'][key]) + to_install.update(Requirement(r) for r in pyproject['project']['optional-dependencies'][key]) cmd = [sys.executable, '-m', 'pip', 'install'] + [str(r) for r in to_install] subprocess.run(cmd, check=True) @@ -407,9 +410,9 @@ def identify_outdated(extras, pre_releases): # Read the requirements from 'pyproject.toml'' pyproject = _load_pyproject() - to_install = {Requirement.parse(r) for r in pyproject['project']['dependencies']} + to_install = {Requirement(r) for r in pyproject['project']['dependencies']} for key in extras: - to_install.update(Requirement.parse(r) for r in pyproject['project']['optional-dependencies'][key]) + to_install.update(Requirement(r) for r in pyproject['project']['optional-dependencies'][key]) def get_package_data(name): req = requests.get(f'https://pypi.python.org/pypi/{name}/json', timeout=5) diff --git a/utils/requirements.txt b/utils/requirements.txt index eb8351b949..7fb8a640c2 100644 --- a/utils/requirements.txt +++ b/utils/requirements.txt @@ -1,5 +1,5 @@ click==7.1.2 -packaging==20.3 -pyyaml==5.4.1 +packaging==23.1 +pyyaml==6.0.1 requests==2.25.1 tomli==2.0.0