Remove CI image build hacks

Fix CI image build issues that made certain hacks necessary
for things to work, and remove hacks that are no longer required
with the new Ubuntu 24.04-based CI image.

* Use the Ubuntu-packaged version of Tox, Setuptools and
  and all other packages installed outside of the test environment.
  This is necessary to make dependency conflicts less likely
  due to setting `sitepackages` to `True` in Tox to pass through
  `python3-rados` to the test environment.
* Remove the hack for deleting the distro-managed version of `pyparsing`,
  as the new version available in Ubuntu 24.04 is compatible.
* Remove the hack for deleting the distro-packaged versions of
  the `jaraco.*` series of packages. The reason why this started
  causing problems is [Setuptools v71.0](https://setuptools.pypa.io/en/latest/history.html#v71-0-0)
  onwards changed behaviour to preferring external dependencies instead
  of the bundled versions if they are installed, even if they are not compatible.
  The `ceph-mgr` package installs the distro-managed version of `jaraco.text`
  (among others), causing the conflict.
* Remove the patch for the distro-managed version of `werkzeug`,
  as the new version now adds the line for deferred annotation validation.
* Don't set `VIRTUALENV_SETUPTOOLS=bundle` explicitly,
  as this is the default behaviour in virtualenvs created by Tox.
  `setuptools` was (and is) actually being installed, it was just
  erroring out on import due to the incompatible version of `jaraco.text`
  being found. `pip`'s error handling was hiding the real cause
  of the failure.

.github/workflows/gnocchi.yml

@@ -167,12 +167,18 @@ jobs:
           push: false
           tags: ghcr.io/gnocchixyz/ci:latest
         if: steps.changes.outputs.ci_image == 'true'
+      # NOTE(callumdickinson): SETUPTOOLS_USE_DISTUTILS needs to be set to 'stdlib'
+      # for Debian-packaged setuptools to work correctly on Python 3.9 and 3.11.
+      # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252
       - name: Set env vars
         run: |
+          export DOCKER_ENV_OPTS=
           if [ "${{ github.event.inputs.debug }}" == "true" ]; then
-            echo "DOCKER_ENV_OPTS=-e GNOCCHI_TEST_DEBUG=1" >> $GITHUB_ENV
-          else
-            echo "DOCKER_ENV_OPTS=" >> $GITHUB_ENV
+            export DOCKER_ENV_OPTS="${DOCKER_ENV_OPTS} -e GNOCCHI_TEST_DEBUG=1"
+          fi
+          if [ "${{ matrix.python }}" = "py39" -o "${{ matrix.python}}" = "py311" ]; then
+            export DOCKER_ENV_OPTS="$DOCKER_ENV_OPTS -e SETUPTOOLS_USE_DISTUTILS=stdlib"
           fi
+          echo "DOCKER_ENV_OPTS=$DOCKER_ENV_OPTS" >> $GITHUB_ENV
       - name: Run tests with tox in container
         run: docker run --rm -v ${{ github.workspace }}:/github/workspace -w /github/workspace $DOCKER_ENV_OPTS ghcr.io/gnocchixyz/ci:latest "tox -e ${{ matrix.python }}-${{ matrix.env }}"

images/Dockerfile.ci

@@ -15,12 +15,16 @@ RUN apt-get update -y && apt-get install -qy \
         python3 \
         python3-dev \
         python3-pip \
+        python3-setuptools \
+        python3-wheel \
         python3-virtualenv \
         python3.9 \
         python3.9-dev \
         python3.9-distutils \
         python3.11 \
         python3.11-dev \
+        python3.11-distutils \
+        tox \
 # Needed for uwsgi core routing support
         build-essential \
         libffi-dev \
@@ -42,15 +46,6 @@ RUN apt-get update -y && apt-get install -qy \
 
 RUN rm -rf /var/lib/apt/lists/*
 
-# NOTE(tobias.urdin): hack since pyparsing in site-packages collides with our requirements
-RUN rm -rf /usr/lib/python3/dist-packages/pyparsing*
-
-# NOTE(tobias.urdin): hack since jaraco packages in site-packages collides with our requirements
-RUN rm -rf /usr/lib/python3/dist-packages/jaraco*
-
-# TODO(tobias.urdin): hack remove this when we drop python 3.9
-RUN sed -i '1s/^/from __future__ import annotations\n/' /usr/lib/python3/dist-packages/werkzeug/sansio/utils.py
-
 #NOTE(sileht): really no utf-8 in 2017 !?
 ENV LANG en_US.UTF-8
 RUN update-locale

images/entrypoint.sh.ci

@@ -1,7 +1,3 @@
 #!/bin/sh
 
-python3 -m virtualenv /tmp/gnocchi-tox-env
-. /tmp/gnocchi-tox-env/bin/activate
-pip install tox
-
 $@

tox.ini

@@ -10,7 +10,7 @@ passenv =
     GNOCCHI_TEST_*
     AWS_*
 setenv =
-    VIRTUALENV_SETUPTOOLS=bundle
+    SETUPTOOLS_USE_DISTUTILS={env:SETUPTOOLS_USE_DISTUTILS:local}
     GNOCCHI_TEST_STORAGE_DRIVER=file
     GNOCCHI_TEST_INDEXER_DRIVER=postgresql
     GNOCCHI_TEST_STORAGE_DRIVERS=file swift ceph s3 redis
@@ -69,7 +69,6 @@ deps =
 # Gnocchi we can't reuse the virtualenv
 recreate = True
 setenv =
-    VIRTUALENV_SETUPTOOLS=bundle
     GNOCCHI_VERSION_FROM=stable/4.5
     GNOCCHI_VARIANT=test,postgresql,file
 deps =
@@ -84,7 +83,6 @@ allowlist_externals = {toxinidir}/run-upgrade-tests.sh
 # Gnocchi we can't reuse the virtualenv
 recreate = True
 setenv =
-    VIRTUALENV_SETUPTOOLS=bundle
     GNOCCHI_VERSION_FROM=stable/4.5
     GNOCCHI_VARIANT=test,mysql,ceph,ceph_recommended_lib
 deps =
@@ -134,7 +132,6 @@ deps =
     .[test,file,postgresql,doc]
     doc8
 setenv =
-    VIRTUALENV_SETUPTOOLS=bundle
     GNOCCHI_TEST_DEBUG=1
 commands =
     doc8 --ignore-path doc/source/rest.rst,doc/source/comparison-table.rst doc/source
@@ -145,7 +142,6 @@ allowlist_externals =
     /bin/bash
     /bin/rm
 setenv =
-    VIRTUALENV_SETUPTOOLS=bundle
     GNOCCHI_STORAGE_DEPS=file
     GNOCCHI_TEST_DEBUG=1
 deps =