Merge pull request #240 from stdweird/esnew

test release with py39 setuptool 53 requirement

lib/vsc/install/ci.py

@@ -37,7 +37,10 @@
 import yaml
 
 from pathlib import Path
-from vsc.install.shared_setup import MAX_SETUPTOOLS_VERSION, vsc_setup
+from vsc.install.shared_setup import (
+    MAX_SETUPTOOLS_VERSION_PY36, MAX_SETUPTOOLS_VERSION_PY39,
+    vsc_setup,
+    )
 
 
 JENKINSFILE = 'Jenkinsfile'
@@ -166,51 +169,65 @@ def gen_tox_ini():
         "skipsdist = true",
     ]
 
-    if not vsc_ci_cfg[PY36_TESTS_MUST_PASS]:
-        lines.extend([
+    test36 = [
             '',
             '[testenv:py36]',
-            'ignore_outcome = true'
-        ])
-
-    if not vsc_ci_cfg[PY39_TESTS_MUST_PASS]:
-        lines.extend([
+    ]
+    test39 = [
             '',
             '[testenv:py39]',
-            'ignore_outcome = true'
-        ])
+    ]
 
-    lines.extend([
-        '',
-        '[testenv]',
-        "commands_pre =",
-    ])
+    if not vsc_ci_cfg[PY36_TESTS_MUST_PASS]:
+        test36.append('ignore_outcome = true')
 
-    if vsc_ci_cfg[MOVE_SETUP_CFG]:
-        lines.append("    mv setup.cfg setup.cfg.moved")
+    if not vsc_ci_cfg[PY39_TESTS_MUST_PASS]:
+        test39.append('ignore_outcome = true')
 
-    pip_install_test_deps = vsc_ci_cfg[PIP_INSTALL_TEST_DEPS]
-    if pip_install_test_deps:
-        for dep in pip_install_test_deps.strip().split('\n'):
-            lines.append(f"    pip install {pip_args}'{dep}'")
+    def make_commands_pre(minor, tlines):
+        if minor > 6:
+            tlines.append("setenv = SETUPTOOLS_USE_DISTUTILS=local")
+
+        tlines.extend([
+            "commands_pre =",
+        ])
+        if vsc_ci_cfg[MOVE_SETUP_CFG]:
+            tlines.append("    mv setup.cfg setup.cfg.moved")
+
+        pip_install_test_deps = vsc_ci_cfg[PIP_INSTALL_TEST_DEPS]
+        if pip_install_test_deps:
+            for dep in pip_install_test_deps.strip().split('\n'):
+                tlines.append(f"    pip install {pip_args}'{dep}'")
 
-    lines.extend([
         # install required setuptools version;
         # we need a setuptools < 42.0 for now, since in 42.0 easy_install was changed to use pip when available;
         # it's important to use pip (not easy_install) here, since only pip will actually remove an older
         # already installed setuptools version
-        f"    pip install {pip_args}'setuptools<{MAX_SETUPTOOLS_VERSION}'",
+        if minor > 6:
+            tlines.append(f"    pip install {pip_args}'setuptools<{MAX_SETUPTOOLS_VERSION_PY39}' wheel")
+        else:
+            tlines.append(f"    pip install {pip_args}'setuptools<{MAX_SETUPTOOLS_VERSION_PY36}'")
         # install latest vsc-install release from PyPI;
         # we can't use 'pip install' here, because then we end up with a broken installation because
         # vsc/__init__.py is not installed because we're using pkg_resources.declare_namespace
         # (see https://github.com/pypa/pip/issues/1924)
-        f"    python -m easy_install -U {easy_install_args}vsc-install",
-    ])
+        if minor > 6:
+            tlines.append(f"    python setup.py -q easy_install -v -U {easy_install_args}vsc-install")
+        else:
+            tlines.append(f"    python -m easy_install -U {easy_install_args}vsc-install")
+
+        if vsc_ci_cfg[MOVE_SETUP_CFG]:
+            tlines.append("    mv setup.cfg.moved setup.cfg")
 
-    if vsc_ci_cfg[MOVE_SETUP_CFG]:
-        lines.append("    mv setup.cfg.moved setup.cfg")
+    make_commands_pre(6, test36)
+    make_commands_pre(9, test39)
+
+    lines.extend(test36)
+    lines.extend(test39)
 
     lines.extend([
+        '',
+        '[testenv]',
         "commands = python setup.py test",
         # $USER is not defined in tox environment, so pass it
         # see https://tox.readthedocs.io/en/latest/example/basic.html#passing-down-environment-variables
@@ -299,6 +316,7 @@ def indent(line, level=1):
     python_cmd = 'python3'
 
     if vsc_ci_cfg[EASY_INSTALL_TOX]:
+        # worst case, use 'SETUPTOOLS_USE_DISTUTILS=local python $PREFIX/setup.py -q easy_install -v ' as "easy_install"
         install_cmd = install_cmd.replace('pip3 install', 'python -m easy_install')
         easy_install_args += '-U --user'
         test_cmds.append(f'{install_cmd} {easy_install_args} tox')

lib/vsc/install/shared_setup.py

@@ -41,6 +41,28 @@
 import re
 import builtins
 
+MAX_SETUPTOOLS_VERSION_PY39 = '54.0'
+MAX_SETUPTOOLS_VERSION_PY36 = '42.0'
+
+if sys.version_info.major == 3 and sys.version_info.minor > 6:
+    # Must run before importing setuptools
+    dmod = sys.modules.get('distutils', None)
+    if dmod is not None and 'setuptools/_distutils' not in dmod.__file__:
+        print(f'WARN: distutils already loaded with unexpected path.')
+        print("  If you get this, set 'SETUPTOOLS_USE_DISTUTILS=local' or check the setuptools version >= 53.0")
+
+
+    # only for sys.version_info.minor == 9 (rhel9 setup)
+    MAX_SETUPTOOLS_VERSION = MAX_SETUPTOOLS_VERSION_PY39
+
+    sud = os.environ.get('SETUPTOOLS_USE_DISTUTILS', None)
+    if sud is None:
+        os.environ['SETUPTOOLS_USE_DISTUTILS'] = 'local'
+    elif sud != 'local':
+        print(f"WARN: Found SETUPTOOLS_USE_DISTUTILS in environ with value '{sud}', only tested with 'local'")
+else:
+    MAX_SETUPTOOLS_VERSION = MAX_SETUPTOOLS_VERSION_PY36
+
 import setuptools
 import setuptools.dist
 import setuptools.command.test
@@ -109,6 +131,7 @@ def __init__(self, *args, **kwargs):
             self._log_levels.update(log_levels)
             OrigLog.__init__(self, *args, **kwargs)
 
+        # pylint: disable=arguments-differ
         def _log(self, level, msg, args):
             """Prefix the message with human readable level"""
             newmsg = f"{self._log_levels.get(level, 'UNKNOWN')}: {msg}"
@@ -301,13 +324,50 @@ def get_egg_cache_dir_pyver(self):
         return egg_cache_dir_pyver
 
     setuptools.dist.Distribution.get_egg_cache_dir = get_egg_cache_dir_pyver
-
-# for ancient setuptools version (< 7.0), get_egg_cache_dir is not there yet
-# in that case, just hard remove the existing .eggs directory, to force re-creating it
 else:
-    eggs_dir = os.path.join(os.getcwd(), '.eggs')
-    if os.path.exists(eggs_dir):
-        shutil.rmtree(eggs_dir)
+    # old workaround is not needed anymore, this code was still around in 53
+    print('ERROR: no get_egg_cache_dir found in setuptools.dist.Distribution')
+
+
+# fetch_build_egg was updated in setuptools 42 to use 'from setuptools.installer import fetch_build_egg'
+# however, that one has logic to use pip
+# reverting this code to the pre-42 behaviour
+if hasattr(setuptools.dist.Distribution, 'fetch_build_egg'):
+    setuptools.dist.Distribution._orig_fetch_build_egg = setuptools.dist.Distribution.fetch_build_egg
+
+    # verbatim copy of 41.6.0-1.el8 setuptools.dist code
+    def fetch_build_egg_pyver(self, req):
+        """Fetch an egg needed for building"""
+        from setuptools.command.easy_install import easy_install
+        dist = self.__class__({'script_args': ['easy_install']})
+        opts = dist.get_option_dict('easy_install')
+        opts.clear()
+        opts.update(
+            (k, v)
+            for k, v in self.get_option_dict('easy_install').items()
+            if k in (
+                # don't use any other settings
+                'find_links', 'site_dirs', 'index_url',
+                'optimize', 'site_dirs', 'allow_hosts',
+            ))
+        if self.dependency_links:
+            links = self.dependency_links[:]
+            if 'find_links' in opts:
+                links = opts['find_links'][1] + links
+            opts['find_links'] = ('setup', links)
+        install_dir = self.get_egg_cache_dir()
+        cmd = easy_install(
+            dist, args=["x"], install_dir=install_dir,
+            exclude_scripts=True,
+            always_copy=False, build_directory=None, editable=False,
+            upgrade=False, multi_version=True, no_report=True, user=False
+        )
+        cmd.ensure_finalized()
+        return cmd.easy_install(req)
+
+    setuptools.dist.Distribution.fetch_build_egg = fetch_build_egg_pyver
+else:
+    print('ERROR: no fetch_build_egg found in setuptools.dist.Distribution')
 
 
 class vsc_setup():
@@ -1727,7 +1787,5 @@ def main():
 
     action_target(PACKAGE)
 
-MAX_SETUPTOOLS_VERSION = '42.0'
-
 if __name__ == '__main__':
     main()

test/ci.py

@@ -97,13 +97,19 @@
 envlist = py36,py39
 skipsdist = true
 
+[testenv:py36]
+commands_pre =
+    pip install 'setuptools<42.0'
+    python -m easy_install -U vsc-install
+
 [testenv:py39]
 ignore_outcome = true
+setenv = SETUPTOOLS_USE_DISTUTILS=local
+commands_pre =
+    pip install 'setuptools<54.0' wheel
+    python setup.py -q easy_install -v -U vsc-install
 
 [testenv]
-commands_pre =
-    pip install 'setuptools<42.0'
-    python -m easy_install -U vsc-install
 commands = python setup.py test
 passenv = USER
 """
@@ -116,10 +122,18 @@
 envlist = py36,py39
 skipsdist = true
 
-[testenv]
+[testenv:py36]
 commands_pre =
     pip install 'setuptools<42.0'
     python -m easy_install -U vsc-install
+
+[testenv:py39]
+setenv = SETUPTOOLS_USE_DISTUTILS=local
+commands_pre =
+    pip install 'setuptools<54.0' wheel
+    python setup.py -q easy_install -v -U vsc-install
+
+[testenv]
 commands = python setup.py test
 passenv = USER
 """
@@ -134,11 +148,17 @@
 
 [testenv:py36]
 ignore_outcome = true
-
-[testenv]
 commands_pre =
     pip install 'setuptools<42.0'
     python -m easy_install -U vsc-install
+
+[testenv:py39]
+setenv = SETUPTOOLS_USE_DISTUTILS=local
+commands_pre =
+    pip install 'setuptools<54.0' wheel
+    python setup.py -q easy_install -v -U vsc-install
+
+[testenv]
 commands = python setup.py test
 passenv = USER
 """
@@ -335,8 +355,8 @@ def test_install_scripts_prefix_override(self):
         pip_regex = re.compile('pip install')
         pip_install_scripts = 'pip install --install-option="--install-scripts={envdir}/bin"'
         expected_tox_ini = pip_regex.sub(pip_install_scripts, expected_tox_ini)
-        easy_install_regex = re.compile('easy_install -U')
-        expected_tox_ini = easy_install_regex.sub('easy_install -U --script-dir={envdir}/bin', expected_tox_ini)
+        easy_install_regex = re.compile('easy_install( -v)? -U')
+        expected_tox_ini = easy_install_regex.sub(r'easy_install\1 -U --script-dir={envdir}/bin', expected_tox_ini)
 
         self.assertEqual(gen_tox_ini(), expected_tox_ini)
 
@@ -352,9 +372,9 @@ def test_move_setup_cfg(self):
             'commands_pre =',
             '    mv setup.cfg setup.cfg.moved',
         ]))
-        expected_tox_ini = expected_tox_ini.replace('commands =', '\n'.join([
+        expected_tox_ini = expected_tox_ini.replace('vsc-install', '\n'.join([
+            'vsc-install',
             '    mv setup.cfg.moved setup.cfg',
-            'commands =',
         ]))
 
         self.assertEqual(gen_tox_ini(), expected_tox_ini)

tox.ini

@@ -6,9 +6,17 @@
 envlist = py36,py39
 skipsdist = true
 
-[testenv]
+[testenv:py36]
 commands_pre =
     pip install 'setuptools<42.0'
     python -m easy_install -U vsc-install
+
+[testenv:py39]
+setenv = SETUPTOOLS_USE_DISTUTILS=local
+commands_pre =
+    pip install 'setuptools<54.0' wheel
+    python setup.py -q easy_install -v -U vsc-install
+
+[testenv]
 commands = python setup.py test
 passenv = USER