From 10674e35d62280903fc948f71300712df83c8ef5 Mon Sep 17 00:00:00 2001 From: Callum Dickinson Date: Wed, 23 Oct 2024 14:36:35 +1300 Subject: [PATCH] Remove CI image build hacks Fix CI image build issues that made certain hacks necessary for things to work, and remove hacks that are no longer required with the new Ubuntu 24.04-based CI image. * Use the Ubuntu-packaged version of Tox, Setuptools and and all other packages installed outside of the test environment. This is necessary to make dependency conflicts less likely due to setting `sitepackages` to `True` in Tox to pass through `python3-rados` to the test environment. * Remove the hack for deleting the distro-managed version of `pyparsing`, as the new version available in Ubuntu 24.04 is compatible. * Remove the hack for deleting the distro-packaged versions of the `jaraco.*` series of packages. The reason why this started causing problems is [Setuptools v71.0](https://setuptools.pypa.io/en/latest/history.html#v71-0-0) onwards changed behaviour to preferring external dependencies instead of the bundled versions if they are installed, even if they are not compatible. The `ceph-mgr` package installs the distro-managed version of `jaraco.text` (among others), causing the conflict. * Remove the patch for the distro-managed version of `werkzeug`, as the new version now adds the line for deferred annotation validation. * Don't set `VIRTUALENV_SETUPTOOLS=bundle` explicitly, as this is the default behaviour in virtualenvs created by Tox. `setuptools` was (and is) actually being installed, it was just erroring out on import due to the incompatible version of `jaraco.text` being found. `pip`'s error handling was hiding the real cause of the failure. --- .github/workflows/gnocchi.yml | 12 +++++++++--- images/Dockerfile.ci | 13 ++++--------- images/entrypoint.sh.ci | 4 ---- tox.ini | 6 +----- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/.github/workflows/gnocchi.yml b/.github/workflows/gnocchi.yml index 1542e671d..282cc0885 100644 --- a/.github/workflows/gnocchi.yml +++ b/.github/workflows/gnocchi.yml @@ -167,12 +167,18 @@ jobs: push: false tags: ghcr.io/gnocchixyz/ci:latest if: steps.changes.outputs.ci_image == 'true' + # NOTE(callumdickinson): SETUPTOOLS_USE_DISTUTILS needs to be set to 'stdlib' + # for Debian-packaged setuptools to work correctly on Python 3.9 and 3.11. + # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252 - name: Set env vars run: | + export DOCKER_ENV_OPTS= if [ "${{ github.event.inputs.debug }}" == "true" ]; then - echo "DOCKER_ENV_OPTS=-e GNOCCHI_TEST_DEBUG=1" >> $GITHUB_ENV - else - echo "DOCKER_ENV_OPTS=" >> $GITHUB_ENV + export DOCKER_ENV_OPTS="${DOCKER_ENV_OPTS} -e GNOCCHI_TEST_DEBUG=1" + fi + if [ "${{ matrix.python }}" = "py39" -o "${{ matrix.python}}" = "py311" ]; then + export DOCKER_ENV_OPTS="$DOCKER_ENV_OPTS -e SETUPTOOLS_USE_DISTUTILS=stdlib" fi + echo "DOCKER_ENV_OPTS=$DOCKER_ENV_OPTS" >> $GITHUB_ENV - name: Run tests with tox in container run: docker run --rm -v ${{ github.workspace }}:/github/workspace -w /github/workspace $DOCKER_ENV_OPTS ghcr.io/gnocchixyz/ci:latest "tox -e ${{ matrix.python }}-${{ matrix.env }}" diff --git a/images/Dockerfile.ci b/images/Dockerfile.ci index 87bf92442..5a0345d07 100644 --- a/images/Dockerfile.ci +++ b/images/Dockerfile.ci @@ -15,12 +15,16 @@ RUN apt-get update -y && apt-get install -qy \ python3 \ python3-dev \ python3-pip \ + python3-setuptools \ + python3-wheel \ python3-virtualenv \ python3.9 \ python3.9-dev \ python3.9-distutils \ python3.11 \ python3.11-dev \ + python3.11-distutils \ + tox \ # Needed for uwsgi core routing support build-essential \ libffi-dev \ @@ -42,15 +46,6 @@ RUN apt-get update -y && apt-get install -qy \ RUN rm -rf /var/lib/apt/lists/* -# NOTE(tobias.urdin): hack since pyparsing in site-packages collides with our requirements -RUN rm -rf /usr/lib/python3/dist-packages/pyparsing* - -# NOTE(tobias.urdin): hack since jaraco packages in site-packages collides with our requirements -RUN rm -rf /usr/lib/python3/dist-packages/jaraco* - -# TODO(tobias.urdin): hack remove this when we drop python 3.9 -RUN sed -i '1s/^/from __future__ import annotations\n/' /usr/lib/python3/dist-packages/werkzeug/sansio/utils.py - #NOTE(sileht): really no utf-8 in 2017 !? ENV LANG en_US.UTF-8 RUN update-locale diff --git a/images/entrypoint.sh.ci b/images/entrypoint.sh.ci index f5e424e49..cc00ea2cb 100755 --- a/images/entrypoint.sh.ci +++ b/images/entrypoint.sh.ci @@ -1,7 +1,3 @@ #!/bin/sh -python3 -m virtualenv /tmp/gnocchi-tox-env -. /tmp/gnocchi-tox-env/bin/activate -pip install tox - $@ diff --git a/tox.ini b/tox.ini index 03f7e9afd..db8acdba0 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ passenv = GNOCCHI_TEST_* AWS_* setenv = - VIRTUALENV_SETUPTOOLS=bundle + SETUPTOOLS_USE_DISTUTILS={env:SETUPTOOLS_USE_DISTUTILS:local} GNOCCHI_TEST_STORAGE_DRIVER=file GNOCCHI_TEST_INDEXER_DRIVER=postgresql GNOCCHI_TEST_STORAGE_DRIVERS=file swift ceph s3 redis @@ -69,7 +69,6 @@ deps = # Gnocchi we can't reuse the virtualenv recreate = True setenv = - VIRTUALENV_SETUPTOOLS=bundle GNOCCHI_VERSION_FROM=stable/4.5 GNOCCHI_VARIANT=test,postgresql,file deps = @@ -84,7 +83,6 @@ allowlist_externals = {toxinidir}/run-upgrade-tests.sh # Gnocchi we can't reuse the virtualenv recreate = True setenv = - VIRTUALENV_SETUPTOOLS=bundle GNOCCHI_VERSION_FROM=stable/4.5 GNOCCHI_VARIANT=test,mysql,ceph,ceph_recommended_lib deps = @@ -134,7 +132,6 @@ deps = .[test,file,postgresql,doc] doc8 setenv = - VIRTUALENV_SETUPTOOLS=bundle GNOCCHI_TEST_DEBUG=1 commands = doc8 --ignore-path doc/source/rest.rst,doc/source/comparison-table.rst doc/source @@ -145,7 +142,6 @@ allowlist_externals = /bin/bash /bin/rm setenv = - VIRTUALENV_SETUPTOOLS=bundle GNOCCHI_STORAGE_DEPS=file GNOCCHI_TEST_DEBUG=1 deps =