Dependencies: Add support for Python 3.12

The file `requirements/requirements-py-3.12.txt` is added which provides
a complete environment for Python 3.12. The CI is updated to add Python
3.12 in all strategy matrices or replace Python 3.11 where only the
oldest and latest Python version are tested. Note that the Python
version for the `pre-commit` jobs are kept at 3.10 for now. The reason
is that in Python 3.12 f-strings are improved by allowing nested quotes.
For example:

    f'some_dict['key']'

is now supported, whereas before Python 3.12 this would not work since
the nested quotes would not be parsed correctly and the internal quotes
had to be either escaped or changed for double quotes.

A number of dependencies had to be updated to make them compatible with
Python 3.12, usually because older version still relied on the
`distutils` and `pkg_resources` standard lib modules which have been
removed. The `utils/dependency_management.py` had to be updated similarly
to replace `pkg_resources` with `packaging`. The latter had to be
updated to `packaging==23.0` in order to have the `__eq__`
implementation for the `Requirement` class which the script relies on.

The memory leak tests are skipped on Python 3.12 because currently they
hang. The problem is with the `pympler.muppy.get_objects` method. This
calls `gc.collect` internally, but that call is blocking. The exact
cause is as of yet unknown. The garbage collecting has been changed in
Python 3.12 so it is not completely unexpected either.

The `sphinxcontrib-details-directive` dependency is removed. It was used
for the sphinx extension to add the ports of port namespaces in HTML's
`<details>` tags, allowing them to be collapsed. This could help with
readability in case of large namespaces. However, this package breaks on
Python 3.12 since it imports the deprecated `pkg_resources` package.
Since the package has not been maintained since 4 years, it is unlikely
this will be fixed it and so instead it is removed for now. See
https://github.com/sphinx-contrib/sphinxcontrib-details-directive

.github/system_tests/pytest/test_memory_leaks.py

@@ -8,6 +8,10 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """Utilities for testing memory leakage."""
+import sys
+
+import pytest
+
 from aiida import orm
 from aiida.engine import processes, run_get_node
 from aiida.plugins import CalculationFactory
@@ -23,6 +27,7 @@ def run_finished_ok(*args, **kwargs):
     assert node.is_finished_ok, (node.exit_status, node.exit_message)
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 12), reason='Garbage collecting hangs on Python 3.12')
 def test_leak_run_process():
     """Test whether running a dummy process leaks memory."""
     inputs = {'a': orm.Int(2), 'b': orm.Str('test')}
@@ -34,6 +39,7 @@ def test_leak_run_process():
     assert not process_instances, f'Memory leak: process instances remain in memory: {process_instances}'
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 12), reason='Garbage collecting hangs on Python 3.12')
 def test_leak_local_calcjob(aiida_local_code_factory):
     """Test whether running a local CalcJob leaks memory."""
     inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': aiida_local_code_factory('core.arithmetic.add', '/bin/bash')}
@@ -45,6 +51,7 @@ def test_leak_local_calcjob(aiida_local_code_factory):
     assert not process_instances, f'Memory leak: process instances remain in memory: {process_instances}'
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 12), reason='Garbage collecting hangs on Python 3.12')
 def test_leak_ssh_calcjob():
     """Test whether running a CalcJob over SSH leaks memory.
 

.github/workflows/ci-code.yml

@@ -23,10 +23,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.12
       uses: actions/setup-python@v2
       with:
-        python-version: '3.9'
+        python-version: '3.12'
 
     - name: Install utils/ dependencies
       run: pip install -r utils/requirements.txt
@@ -55,7 +55,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.11']
+        python-version: ['3.9', '3.12']
 
     services:
       postgres:
@@ -131,7 +131,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.11']
+        python-version: ['3.9', '3.12']
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/test-install.yml

@@ -58,7 +58,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v2
@@ -144,7 +144,7 @@ jobs:
       fail-fast: false
       matrix:
 
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
         # Not being able to install with conda on a specific Python version is
         # not sufficient to fail the run, but something we want to be aware of.
@@ -195,7 +195,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     services:
       postgres:

aiida/sphinxext/__init__.py

@@ -16,7 +16,6 @@ def setup(app):
 
     from . import calcjob, process, workchain
 
-    app.setup_extension('sphinxcontrib.details.directive')
     process.setup_extension(app)
     workchain.setup_extension(app)
     calcjob.setup_extension(app)

aiida/sphinxext/process.py

@@ -20,7 +20,6 @@
 from sphinx import addnodes
 from sphinx.ext.autodoc import ClassDocumenter
 from sphinx.util.docutils import SphinxDirective
-from sphinxcontrib.details.directive import details, summary
 
 from aiida.common.utils import get_object_from_string
 from aiida.engine import Process
@@ -156,12 +155,7 @@ def build_portnamespace_doctree(self, port_namespace):
                 if port.help is not None:
                     item += nodes.Text(' -- ')
                     item.extend(publish_doctree(port.help)[0].children)
-                sub_doctree = self.build_portnamespace_doctree(port)
-                if sub_doctree:
-                    sub_item = details(opened=self.EXPAND_NAMESPACES_FLAG in self.options)
-                    sub_item += summary(text='Namespace Ports')
-                    sub_item += sub_doctree
-                    item += sub_item
+                item += self.build_portnamespace_doctree(port)
             else:
                 raise NotImplementedError
             result += item

pyproject.toml

@@ -19,6 +19,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering"
 ]
 keywords = ["aiida", "workflows"]
@@ -78,15 +79,14 @@ rest = [
 ]
 docs = [
     "pydata-sphinx-theme~=0.13.3",
-    "sphinx~=4.1",
-    "sphinxcontrib-details-directive~=0.1.0",
+    "sphinx~=7.2",
     "sphinx-copybutton~=0.5.0",
-    "sphinx-design~=0.0.13",
-    "sphinx-notfound-page~=0.5",
+    "sphinx-design~=0.5.0",
+    "sphinx-notfound-page~=1.0",
     "sphinxext-rediraffe~=0.2.4",
   #  "sphinx-sqlalchemy~=0.1.1",
     "sphinx-intl~=2.1.0",
-    "myst-nb~=0.17.0",
+    "myst-nb~=1.0.0",
 ]
 atomic_tools = [
     "PyCifRW~=4.4",
@@ -104,7 +104,7 @@ notebook = [
 ]
 pre-commit = [
     "mypy==0.991",
-    "packaging~=20.9",
+    "packaging~=23.0",
     "pre-commit~=2.2",
     "pylint~=2.17.4",
     "pylint-aiida~=0.1.1",
@@ -118,15 +118,15 @@ tests = [
     "pgtest~=1.3,>=1.3.1",
     "pytest~=7.0",
     "pytest-asyncio~=0.12,<0.17",
-    "pytest-timeout~=1.3",
+    "pytest-timeout~=2.0",
     "pytest-cov~=2.7,<2.11",
-    "pytest-rerunfailures~=9.1,>=9.1.1",
+    "pytest-rerunfailures~=12.0",
     "pytest-benchmark~=4.0",
     "pytest-regressions~=2.2",
     "pympler~=0.9",
     "coverage~=6.0",
-    "sphinx~=4.0",
-    "docutils==0.16",
+    "sphinx~=7.2",
+    "docutils~=0.20",
 ]
 bpython = [
     "bpython~=0.18.0"

requirements/requirements-py-3.10.txt

@@ -43,7 +43,8 @@ defusedxml==0.7.1
 deprecation==2.1.0
 disk-objectstore==1.0.0
 docstring-parser==0.15
-docutils==0.16
+docutils==0.20.1
+emmet-core==0.57.1
 exceptiongroup==1.1.1
 executing==1.2.0
 fastjsonschema==2.17.1
@@ -81,19 +82,19 @@ kiwisolver==1.4.4
 latexcodec==2.0.1
 linkify-it-py==2.0.2
 mako==1.2.4
-markdown-it-py[linkify,plugins]==2.2.0
+markdown-it-py[linkify,plugins]==3.0.0
 markupsafe==2.1.3
 matplotlib==3.7.1
 matplotlib-inline==0.1.6
-mdit-py-plugins==0.3.5
+mdit-py-plugins==0.4.0
 mdurl==0.1.2
 mistune==3.0.1
 monty==2023.9.25
 mpmath==1.3.0
 msgpack==1.0.5
 multidict==6.0.4
-myst-nb==0.17.2
-myst-parser==0.18.1
+myst-nb==1.0.0
+myst-parser==2.0.0
 nbclassic==1.0.0
 nbclient==0.7.4
 nbconvert==7.6.0
@@ -146,8 +147,8 @@ pytest-benchmark==4.0.0
 pytest-cov==2.10.1
 pytest-datadir==1.4.1
 pytest-regressions==2.4.2
-pytest-rerunfailures==9.1.1
-pytest-timeout==1.4.2
+pytest-rerunfailures==12.0.0
+pytest-timeout==2.2.0
 python-dateutil==2.8.2
 python-json-logger==2.0.7
 python-memcached==1.59
@@ -173,18 +174,17 @@ sniffio==1.3.0
 snowballstemmer==2.2.0
 soupsieve==2.4.1
 spglib==2.0.2
-sphinx==4.5.0
+sphinx==7.2.6
 sphinx-copybutton==0.5.2
-sphinx-design==0.0.13
+sphinx-design==0.5.0
 sphinx-intl==2.1.0
-sphinx-notfound-page==0.8.3
+sphinx-notfound-page==1.0.0
 sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-details-directive==0.1.0
 sphinxcontrib-devhelp==1.0.2
 sphinxcontrib-htmlhelp==2.0.1
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-serializinghtml==1.1.9
 sphinxext-rediraffe==0.2.7
 sqlalchemy==2.0.23
 stack-data==0.6.2

requirements/requirements-py-3.11.txt

@@ -43,7 +43,8 @@ defusedxml==0.7.1
 deprecation==2.1.0
 disk-objectstore==1.0.0
 docstring-parser==0.15
-docutils==0.16
+docutils==0.20.1
+emmet-core==0.57.1
 executing==1.2.0
 fastjsonschema==2.17.1
 flask==2.3.2
@@ -80,19 +81,19 @@ kiwisolver==1.4.4
 latexcodec==2.0.1
 linkify-it-py==2.0.2
 mako==1.2.4
-markdown-it-py[linkify,plugins]==2.2.0
+markdown-it-py[linkify,plugins]==3.0.0
 markupsafe==2.1.3
 matplotlib==3.7.1
 matplotlib-inline==0.1.6
-mdit-py-plugins==0.3.5
+mdit-py-plugins==0.4.0
 mdurl==0.1.2
 mistune==3.0.1
 monty==2023.9.25
 mpmath==1.3.0
 msgpack==1.0.5
 multidict==6.0.4
-myst-nb==0.17.2
-myst-parser==0.18.1
+myst-nb==1.0.0
+myst-parser==2.0.0
 nbclassic==1.0.0
 nbclient==0.7.4
 nbconvert==7.6.0
@@ -145,8 +146,8 @@ pytest-benchmark==4.0.0
 pytest-cov==2.10.1
 pytest-datadir==1.4.1
 pytest-regressions==2.4.2
-pytest-rerunfailures==9.1.1
-pytest-timeout==1.4.2
+pytest-rerunfailures==12.0.0
+pytest-timeout==2.2.0
 python-dateutil==2.8.2
 python-json-logger==2.0.7
 python-memcached==1.59
@@ -172,18 +173,17 @@ sniffio==1.3.0
 snowballstemmer==2.2.0
 soupsieve==2.4.1
 spglib==2.0.2
-sphinx==4.5.0
+sphinx==7.2.6
 sphinx-copybutton==0.5.2
-sphinx-design==0.0.13
+sphinx-design==0.5.0
 sphinx-intl==2.1.0
-sphinx-notfound-page==0.8.3
+sphinx-notfound-page==1.0.0
 sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-details-directive==0.1.0
 sphinxcontrib-devhelp==1.0.2
 sphinxcontrib-htmlhelp==2.0.1
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-serializinghtml==1.1.9
 sphinxext-rediraffe==0.2.7
 sqlalchemy==2.0.23
 stack-data==0.6.2