From f66a161abed4e691f74f4c293adaac4a23ae7a15 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Mon, 13 Feb 2023 17:46:03 +0000 Subject: [PATCH 01/39] Initial commit with base functionality. --- setup.cfg | 2 +- tox_conda/env_activator.py | 128 -------------- tox_conda/plugin.py | 339 ++++++++++++++++++++++++++++++++----- 3 files changed, 295 insertions(+), 174 deletions(-) delete mode 100644 tox_conda/env_activator.py diff --git a/setup.cfg b/setup.cfg index 79a5926..b8a9bb3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,7 @@ classifiers = packages = find: install_requires = ruamel.yaml>=0.15.0,<0.18 - tox>=3.8.1,<4 + tox>=4 python_requires = >=3.5 [options.packages.find] diff --git a/tox_conda/env_activator.py b/tox_conda/env_activator.py deleted file mode 100644 index 1ebe855..0000000 --- a/tox_conda/env_activator.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Wrap the tox command for subprocess to activate the target anaconda env.""" -import abc -import os -import shlex -import tempfile -from contextlib import contextmanager - -import tox - - -class PopenInActivatedEnvBase(abc.ABC): - """A base functor that wraps popen calls in an activated anaconda env.""" - - def __init__(self, venv, popen): - self._venv = venv - self.__popen = popen - - def __call__(self, cmd_args, **kwargs): - wrapped_cmd_args = self._wrap_cmd_args(cmd_args) - return self.__popen(wrapped_cmd_args, **kwargs) - - @abc.abstractmethod - def _wrap_cmd_args(self, cmd_args): - """Return the wrapped command arguments.""" - - -class PopenInActivatedEnvPosix(PopenInActivatedEnvBase): - """Wrap popen calls in an activated anaconda env for POSIX platforms. - - The command line to be executed are written to a temporary shell script. - The shell script first activates the env. - """ - - def __init__(self, venv, popen): - super().__init__(venv, popen) - self.__tmp_file = None - - def _wrap_cmd_args(self, cmd_args): - conda_exe = shlex.quote(str(self._venv.envconfig.conda_exe)) - envdir = shlex.quote(str(self._venv.envconfig.envdir)) - - conda_activate_cmd = 'eval "$({conda_exe} shell.posix activate {envdir})"'.format( - conda_exe=conda_exe, envdir=envdir - ) - - # Get a temporary file path. - with tempfile.NamedTemporaryFile() as fp: - self.__tmp_file = fp.name - - # Convert the command args to a command line. - cmd_line = " ".join(map(shlex.quote, cmd_args)) - - with open(self.__tmp_file, "w") as fp: - fp.writelines((conda_activate_cmd, "\n", cmd_line)) - - return ["/bin/sh", self.__tmp_file] - - def __del__(self): - # Delete the eventual temporary script. - if self.__tmp_file is not None: - os.remove(self.__tmp_file) - - -class PopenInActivatedEnvWindows(PopenInActivatedEnvBase): - """Wrap popen call in an activated anaconda env for Windows. - - The shell is temporary forced to cmd.exe and the env is activated accordingly. - This works without a script, the env activation command and the target - command line are concatenated into a single command line. - """ - - def __call__(self, cmd_args, **kwargs): - # Backup COMSPEC before setting it to cmd.exe. - old_comspec = os.environ.get("COMSPEC") - self.__ensure_comspecs_is_cmd_exe() - - output = super().__call__(cmd_args, **kwargs) - - # Revert COMSPEC to its initial value. - if old_comspec is None: - del os.environ["COMSPEC"] - else: - os.environ["COMSPEC"] = old_comspec - - return output - - def _wrap_cmd_args(self, cmd_args): - return ["conda.bat", "activate", str(self._venv.envconfig.envdir), "&&"] + cmd_args - - def __ensure_comspecs_is_cmd_exe(self): - if os.path.basename(os.environ.get("COMSPEC", "")).lower() == "cmd.exe": - return - - for env_var in ("SystemRoot", "windir"): - root_path = os.environ.get(env_var) - if root_path is None: - continue - cmd_exe = os.path.join(root_path, "System32", "cmd.exe") - if os.path.isfile(cmd_exe): - os.environ["COMSPEC"] = cmd_exe - break - else: - tox.reporter.error("cmd.exe cannot be found") - raise SystemExit(0) - - -if tox.INFO.IS_WIN: - PopenInActivatedEnv = PopenInActivatedEnvWindows -else: - PopenInActivatedEnv = PopenInActivatedEnvPosix - - -@contextmanager -def activate_env(venv, action=None): - """Run a command in a temporary activated anaconda env.""" - if action is None: - initial_popen = venv.popen - venv.popen = PopenInActivatedEnv(venv, initial_popen) - else: - initial_popen = action.via_popen - action.via_popen = PopenInActivatedEnv(venv, initial_popen) - - yield - - if action is None: - venv.popen = initial_popen - else: - action.via_popen = initial_popen diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index f93631f..a5dfb5a 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -5,24 +5,264 @@ import subprocess import tempfile from pathlib import Path +from functools import partial -import pluggy -import py.path -import tox -from ruamel.yaml import YAML -from tox.config import DepConfig, DepOption, TestenvConfig -from tox.venv import VirtualEnv -from .env_activator import activate_env +MISSING_CONDA_ERROR = "Cannot locate the conda executable." -hookimpl = pluggy.HookimplMarker("tox") -MISSING_CONDA_ERROR = "Cannot locate the conda executable." +import json +import os +import re +import shlex +import shutil +import subprocess +from io import BytesIO, TextIOWrapper +from pathlib import Path +from time import sleep +from typing import Any, Dict, List + +from tox.execute.api import ( + Execute, + ExecuteInstance, + ExecuteOptions, + ExecuteRequest, + SyncWrite, +) +from tox.execute.local_sub_process import ( + LocalSubProcessExecuteInstance, + LocalSubProcessExecutor, +) +from tox.plugin import impl +from tox.plugin.spec import EnvConfigSet, State, ToxEnvRegister, ToxParser +from tox.tox_env.api import StdinSource, ToxEnv, ToxEnvCreateArgs +from tox.tox_env.errors import Fail +from tox.tox_env.installer import Installer +from tox.tox_env.python.pip.pip_install import Pip +from tox.tox_env.python.virtual_env.runner import VirtualEnvRunner +from tox.tox_env.python.pip.req_file import PythonDeps + + +class CondaEnvRunner(VirtualEnvRunner): + def __init__(self, create_args: ToxEnvCreateArgs) -> None: + self._installer = None + self._executor = None + self._created = False + super().__init__(create_args) + + @staticmethod + def id() -> str: # noqa A003 + return "conda" + + def _get_python_env_version(self): + # Try to use base_python config + match = re.match( + r"python(\d)(?:\.(\d+))?(?:\.?(\d))?", self.conf["base_python"][0] + ) + if match: + groups = match.groups() + version = groups[0] + if groups[1]: + version += ".{}".format(groups[1]) + if groups[2]: + version += ".{}".format(groups[2]) + return version + else: + return self.base_python.version_dot + + @staticmethod + def _find_conda() -> Path: + # This should work if we're not already in an environment + conda_exe = os.environ.get("_CONDA_EXE") + if conda_exe: + return Path(conda_exe).resolve() + + # This should work if we're in an active environment + conda_exe = os.environ.get("CONDA_EXE") + if conda_exe: + return Path(conda_exe).resolve() + + conda_exe = shutil.which("conda") + if conda_exe: + return Path(conda_exe).resolve() + + raise Fail("Failed to find 'conda' executable.") + + def create_python_env(self) -> None: + conda_exe = CondaEnvRunner._find_conda() + python_version = self._get_python_env_version() + + conf = self.python_cache() + + cmd = f"'{conda_exe}' create {conf['conda_env_spec']} '{conf['conda_env']}' python={python_version} --yes --quiet" + try: + cmd_list = shlex.split(cmd) + subprocess.run(cmd_list, check=True) + except subprocess.CalledProcessError as e: + raise Fail( + f"Failed to create '{self.env_dir}' conda environment. Error: {e}" + ) + + def python_cache(self) -> Dict[str, Any]: + base = super().python_cache() + + conda_name = getattr(self.options, "conda_name", None) + if not conda_name and "conda_name" in self.conf: + conda_name = self.conf["conda_name"] + + if conda_name: + conda_env_spec = "-n" + conda_env = conda_name + else: + conda_env_spec = "-p" + conda_env = f"{self.env_dir}" + + base.update( + {"conda_env_spec": conda_env_spec, "conda_env": conda_env}, + ) + return base + + def _ensure_python_env_exists(self) -> None: + if not Path(self.env_dir).exists(): + self.create_python_env() + self._created = True + return + + if self._created: + return + + conda_exe = CondaEnvRunner._find_conda() + cmd = f"'{conda_exe}' env list --json" + try: + cmd_list = shlex.split(cmd) + result: subprocess.CompletedProcess = subprocess.run( + cmd_list, check=True, capture_output=True + ) + except subprocess.CalledProcessError as e: + raise Fail(f"Failed to list conda environments. Error: {e}") + envs = json.loads(result.stdout.decode()) + if str(self.env_dir) in envs["envs"]: + self._created = True + else: + raise Fail( + f"{self.env_dir} already exists, but it is not a conda environment. Delete in manually first." + ) + + def env_site_package_dir(self) -> Path: + """The site package folder withinn the tox environment.""" + cmd = 'from sysconfig import get_paths; print(get_paths()["purelib"])' + path = self._call_python_in_conda_env(cmd, "env_site_package_dir") + return Path(path).resolve() + + def env_python(self) -> Path: + """The python executable within the tox environment.""" + cmd = "import sys; print(sys.executable)" + path = self._call_python_in_conda_env(cmd, "env_python") + return Path(path).resolve() + + def env_bin_dir(self) -> Path: + """The binary folder within the tox environment.""" + cmd = 'from sysconfig import get_paths; print(get_paths()["scripts"])' + path = self._call_python_in_conda_env(cmd, "env_bin_dir") + return Path(path).resolve() + + @property + def executor(self) -> Execute: + def create_conda_command_prefix(): + conf = self.python_cache() + return [ + "conda", + "run", + conf["conda_env_spec"], + conf["conda_env"], + "--live-stream", + ] + + class CondaExecutor(LocalSubProcessExecutor): + def build_instance( + self, + request: ExecuteRequest, + options: ExecuteOptions, + out: SyncWrite, + err: SyncWrite, + ) -> ExecuteInstance: + conda_cmd = create_conda_command_prefix() + + conda_request = ExecuteRequest( + conda_cmd + request.cmd, + request.cwd, + request.env, + request.stdin, + request.run_id, + request.allow, + ) + return LocalSubProcessExecuteInstance(conda_request, options, out, err) + + if self._executor is None: + self._executor = CondaExecutor(self.options.is_colored) + return self._executor + + def _call_python_in_conda_env(self, cmd: str, run_id: str): + self._ensure_python_env_exists() + + python_cmd = "python -c".split() + + class NamedBytesIO(BytesIO): + def __init__(self, name): + self.name = name + super().__init__() + + out_buffer = NamedBytesIO("output") + out = TextIOWrapper(out_buffer, encoding="utf-8") + + err_buffer = NamedBytesIO("error") + err = TextIOWrapper(err_buffer, encoding="utf-8") + + out_err = out, err + + request = ExecuteRequest( + python_cmd + [cmd], + self.conf["change_dir"], + self.environment_variables, + StdinSource.API, + run_id, + ) + + with self.executor.call(request, True, out_err, self) as execute_status: + while execute_status.wait() is None: + sleep(0.01) + if execute_status.exit_code != 0: + raise Fail( + f"Failed to execute operation '{cmd}'. Stderr: {execute_status.err.decode()}" + ) + + return execute_status.out.decode().strip() + @property + def installer(self) -> Installer[Any]: + if self._installer is None: + self._installer = Pip(self) + return self._installer -class CondaDepOption(DepOption): - name = "conda_deps" - help = "each line specifies a conda dependency in pip/setuptools format" + def prepend_env_var_path(self) -> List[Path]: + conda_exe: Path = CondaEnvRunner._find_conda() + return [conda_exe.parent] + + def _default_pass_env(self) -> List[str]: + env = super()._default_pass_env() + env.append("*CONDA*") + return env + + +@impl +def tox_register_tox_env(register: ToxEnvRegister) -> None: # noqa: U100 + register.add_run_env(CondaEnvRunner) + try: + CondaEnvRunner._find_conda() + if "CONDA_DEFAULT_ENV" in os.environ: + register.default_env_runner = "conda" + except Fail: + pass def postprocess_path_option(testenv_config, value): @@ -55,41 +295,50 @@ def get_py_version(envconfig, action): return "python={}".format(version) -@hookimpl -def tox_addoption(parser): - parser.add_testenv_attribute( - name="conda_env", - type="path", - help="specify a conda environment.yml file", - postprocess=postprocess_path_option, +@impl +def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: + env_conf.add_config( + "conda_env", + of_type="path", + desc="specify a conda environment.yml file", + default=None, + post_process=postprocess_path_option, ) - parser.add_testenv_attribute( - name="conda_spec", - type="path", - help="specify a conda spec-file.txt file", - postprocess=postprocess_path_option, + env_conf.add_config( + "conda_spec", + of_type="path", + desc="specify a conda spec-file.txt file", + default=None, + post_process=postprocess_path_option, ) - parser.add_testenv_attribute_obj(CondaDepOption()) + root = env_conf._conf.core["tox_root"] + env_conf.add_config( + "conda_deps", + of_type=PythonDeps, + factory=partial(PythonDeps.factory, root), + default=PythonDeps("", root), + desc="each line specifies a conda dependency in pip/setuptools format", + ) - parser.add_testenv_attribute( - name="conda_channels", type="line-list", help="each line specifies a conda channel" + env_conf.add_config( + "conda_channels", of_type="line-list", desc="each line specifies a conda channel", default=None, ) - parser.add_testenv_attribute( - name="conda_install_args", - type="line-list", - help="each line specifies a conda install argument", + env_conf.add_config( + "conda_install_args", + of_type="line-list", + desc="each line specifies a conda install argument",default=None, ) - parser.add_testenv_attribute( - name="conda_create_args", - type="line-list", - help="each line specifies a conda create argument", + env_conf.add_config( + "conda_create_args", + of_type="line-list", + desc="each line specifies a conda create argument",default=None, ) -@hookimpl +# @hookimpl def tox_configure(config): # This is a pretty cheesy workaround. It allows tox to consider changes to # the conda dependencies when it decides whether an existing environment @@ -150,7 +399,7 @@ def _run_conda_process(args, venv, action, cwd): venv._pcall(args, venv=False, action=action, cwd=cwd, redirect=redirect) -@hookimpl +# @hookimpl def tox_testenv_create(venv, action): tox.venv.cleanup_for_venv(venv) basepath = venv.path.dirpath() @@ -244,7 +493,7 @@ def install_conda_deps(venv, action, basepath, envdir): _run_conda_process(args, venv, action, basepath) -@hookimpl +# @hookimpl def tox_testenv_install_deps(venv, action): # Save the deps before we make temporary changes. saved_deps = copy.deepcopy(venv.envconfig.deps) @@ -273,7 +522,7 @@ def tox_testenv_install_deps(venv, action): return True -@hookimpl +# @hookimpl def tox_get_python_executable(envconfig): if tox.INFO.IS_WIN: path = envconfig.envdir.join("python.exe") @@ -293,8 +542,8 @@ def get_envpython(self): return self.envdir.join("python") -TestenvConfig.__get_envpython = TestenvConfig.get_envpython -TestenvConfig.get_envpython = get_envpython +# TestenvConfig.__get_envpython = TestenvConfig.get_envpython +# TestenvConfig.get_envpython = get_envpython # Monkey patch TestenvConfig _venv_lookup to fix tox behavior with tox-conda under windows @@ -310,23 +559,23 @@ def venv_lookup(self, name): return py.path.local.sysfind(name, paths=paths) -VirtualEnv._venv_lookup = venv_lookup +# VirtualEnv._venv_lookup = venv_lookup -@hookimpl(hookwrapper=True) +# @hookimpl(hookwrapper=True) def tox_runtest_pre(venv): with activate_env(venv): yield -@hookimpl +# @hookimpl def tox_runtest(venv, redirect): with activate_env(venv): tox.venv.tox_runtest(venv, redirect) return True -@hookimpl(hookwrapper=True) +# @hookimpl(hookwrapper=True) def tox_runtest_post(venv): with activate_env(venv): yield From bb87ba7dd5182cdf41f46ba315e9e681c646ff1f Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Mon, 20 Feb 2023 10:42:15 +0000 Subject: [PATCH 02/39] Merge find_conda functions. --- tox_conda/plugin.py | 51 ++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index a5dfb5a..9865076 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -43,6 +43,8 @@ from tox.tox_env.python.pip.req_file import PythonDeps +__all__ = [] + class CondaEnvRunner(VirtualEnvRunner): def __init__(self, create_args: ToxEnvCreateArgs) -> None: self._installer = None @@ -70,26 +72,8 @@ def _get_python_env_version(self): else: return self.base_python.version_dot - @staticmethod - def _find_conda() -> Path: - # This should work if we're not already in an environment - conda_exe = os.environ.get("_CONDA_EXE") - if conda_exe: - return Path(conda_exe).resolve() - - # This should work if we're in an active environment - conda_exe = os.environ.get("CONDA_EXE") - if conda_exe: - return Path(conda_exe).resolve() - - conda_exe = shutil.which("conda") - if conda_exe: - return Path(conda_exe).resolve() - - raise Fail("Failed to find 'conda' executable.") - def create_python_env(self) -> None: - conda_exe = CondaEnvRunner._find_conda() + conda_exe = find_conda() python_version = self._get_python_env_version() conf = self.python_cache() @@ -131,7 +115,7 @@ def _ensure_python_env_exists(self) -> None: if self._created: return - conda_exe = CondaEnvRunner._find_conda() + conda_exe = find_conda() cmd = f"'{conda_exe}' env list --json" try: cmd_list = shlex.split(cmd) @@ -245,7 +229,7 @@ def installer(self) -> Installer[Any]: return self._installer def prepend_env_var_path(self) -> List[Path]: - conda_exe: Path = CondaEnvRunner._find_conda() + conda_exe: Path = find_conda() return [conda_exe.parent] def _default_pass_env(self) -> List[str]: @@ -258,7 +242,8 @@ def _default_pass_env(self) -> List[str]: def tox_register_tox_env(register: ToxEnvRegister) -> None: # noqa: U100 register.add_run_env(CondaEnvRunner) try: - CondaEnvRunner._find_conda() + # Change the defaukt runner only if conda is available + find_conda() if "CONDA_DEFAULT_ENV" in os.environ: register.default_env_runner = "conda" except Fail: @@ -365,6 +350,28 @@ def tox_configure(config): envconfig.conda_exe = conda_exe +def find_conda() -> Path: + # This should work if we're not already in an environment + conda_exe = os.environ.get("_CONDA_EXE") + if conda_exe: + return Path(conda_exe).resolve() + + # This should work if we're in an active environment + conda_exe = os.environ.get("CONDA_EXE") + if conda_exe: + return Path(conda_exe).resolve() + + conda_exe = shutil.which("conda") + if conda_exe: + conda_exe = Path(conda_exe).resolve() + try: + subprocess.run([str(conda_exe), "-h"], stdout=subprocess.DEVNULL) + return conda_exe + except subprocess.CalledProcessError: + pass + + raise Fail("Failed to find 'conda' executable.") + def find_conda(): # This should work if we're not already in an environment conda_exe = os.environ.get("_CONDA_EXE") From 0fc50681fe1addc42d72c54f0b02067545e47abc Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Mon, 20 Feb 2023 10:55:07 +0000 Subject: [PATCH 03/39] Rearrange CondaEnvRunner. --- tox_conda/plugin.py | 164 +++++++++++++++----------------------------- 1 file changed, 56 insertions(+), 108 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 9865076..8f95b9a 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -106,50 +106,6 @@ def python_cache(self) -> Dict[str, Any]: ) return base - def _ensure_python_env_exists(self) -> None: - if not Path(self.env_dir).exists(): - self.create_python_env() - self._created = True - return - - if self._created: - return - - conda_exe = find_conda() - cmd = f"'{conda_exe}' env list --json" - try: - cmd_list = shlex.split(cmd) - result: subprocess.CompletedProcess = subprocess.run( - cmd_list, check=True, capture_output=True - ) - except subprocess.CalledProcessError as e: - raise Fail(f"Failed to list conda environments. Error: {e}") - envs = json.loads(result.stdout.decode()) - if str(self.env_dir) in envs["envs"]: - self._created = True - else: - raise Fail( - f"{self.env_dir} already exists, but it is not a conda environment. Delete in manually first." - ) - - def env_site_package_dir(self) -> Path: - """The site package folder withinn the tox environment.""" - cmd = 'from sysconfig import get_paths; print(get_paths()["purelib"])' - path = self._call_python_in_conda_env(cmd, "env_site_package_dir") - return Path(path).resolve() - - def env_python(self) -> Path: - """The python executable within the tox environment.""" - cmd = "import sys; print(sys.executable)" - path = self._call_python_in_conda_env(cmd, "env_python") - return Path(path).resolve() - - def env_bin_dir(self) -> Path: - """The binary folder within the tox environment.""" - cmd = 'from sysconfig import get_paths; print(get_paths()["scripts"])' - path = self._call_python_in_conda_env(cmd, "env_bin_dir") - return Path(path).resolve() - @property def executor(self) -> Execute: def create_conda_command_prefix(): @@ -186,6 +142,39 @@ def build_instance( self._executor = CondaExecutor(self.options.is_colored) return self._executor + @property + def installer(self) -> Installer[Any]: + if self._installer is None: + self._installer = Pip(self) + return self._installer + + def prepend_env_var_path(self) -> List[Path]: + conda_exe: Path = find_conda() + return [conda_exe.parent] + + def _default_pass_env(self) -> List[str]: + env = super()._default_pass_env() + env.append("*CONDA*") + return env + + def env_site_package_dir(self) -> Path: + """The site package folder within the tox environment.""" + cmd = 'from sysconfig import get_paths; print(get_paths()["purelib"])' + path = self._call_python_in_conda_env(cmd, "env_site_package_dir") + return Path(path).resolve() + + def env_python(self) -> Path: + """The python executable within the tox environment.""" + cmd = "import sys; print(sys.executable)" + path = self._call_python_in_conda_env(cmd, "env_python") + return Path(path).resolve() + + def env_bin_dir(self) -> Path: + """The binary folder within the tox environment.""" + cmd = 'from sysconfig import get_paths; print(get_paths()["scripts"])' + path = self._call_python_in_conda_env(cmd, "env_bin_dir") + return Path(path).resolve() + def _call_python_in_conda_env(self, cmd: str, run_id: str): self._ensure_python_env_exists() @@ -222,20 +211,31 @@ def __init__(self, name): return execute_status.out.decode().strip() - @property - def installer(self) -> Installer[Any]: - if self._installer is None: - self._installer = Pip(self) - return self._installer + def _ensure_python_env_exists(self) -> None: + if not Path(self.env_dir).exists(): + self.create_python_env() + self._created = True + return - def prepend_env_var_path(self) -> List[Path]: - conda_exe: Path = find_conda() - return [conda_exe.parent] + if self._created: + return - def _default_pass_env(self) -> List[str]: - env = super()._default_pass_env() - env.append("*CONDA*") - return env + conda_exe = find_conda() + cmd = f"'{conda_exe}' env list --json" + try: + cmd_list = shlex.split(cmd) + result: subprocess.CompletedProcess = subprocess.run( + cmd_list, check=True, capture_output=True + ) + except subprocess.CalledProcessError as e: + raise Fail(f"Failed to list conda environments. Error: {e}") + envs = json.loads(result.stdout.decode()) + if str(self.env_dir) in envs["envs"]: + self._created = True + else: + raise Fail( + f"{self.env_dir} already exists, but it is not a conda environment. Delete in manually first." + ) @impl @@ -256,30 +256,6 @@ def postprocess_path_option(testenv_config, value): return value -def get_py_version(envconfig, action): - # Try to use basepython - match = re.match(r"python(\d)(?:\.(\d+))?(?:\.?(\d))?", envconfig.basepython) - if match: - groups = match.groups() - version = groups[0] - if groups[1]: - version += ".{}".format(groups[1]) - if groups[2]: - version += ".{}".format(groups[2]) - - # First fallback - elif envconfig.python_info.version_info: - version = "{}.{}".format(*envconfig.python_info.version_info[:2]) - - # Second fallback - else: - code = "import sys; print('{}.{}'.format(*sys.version_info[:2]))" - result = action.popen([envconfig.basepython, "-c", code], report_fail=True, returnout=True) - version = result.decode("utf-8").strip() - - return "python={}".format(version) - - @impl def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: env_conf.add_config( @@ -372,34 +348,6 @@ def find_conda() -> Path: raise Fail("Failed to find 'conda' executable.") -def find_conda(): - # This should work if we're not already in an environment - conda_exe = os.environ.get("_CONDA_EXE") - if conda_exe: - return conda_exe - - # This should work if we're in an active environment - conda_exe = os.environ.get("CONDA_EXE") - if conda_exe: - return conda_exe - - path = shutil.which("conda") - - if path is None: - _exit_on_missing_conda() - - try: - subprocess.run([str(path), "-h"], stdout=subprocess.DEVNULL) - except subprocess.CalledProcessError: - _exit_on_missing_conda() - - return path - - -def _exit_on_missing_conda(): - tox.reporter.error(MISSING_CONDA_ERROR) - raise SystemExit(0) - def _run_conda_process(args, venv, action, cwd): redirect = tox.reporter.verbosity() < tox.reporter.Verbosity.DEBUG From 48c40ca22595e40e878c36eb4c54ccb276e7d9dc Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Mon, 20 Feb 2023 14:54:43 +0000 Subject: [PATCH 04/39] Implement conda_env. --- tox_conda/plugin.py | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 8f95b9a..0f5b8eb 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -75,10 +75,33 @@ def _get_python_env_version(self): def create_python_env(self) -> None: conda_exe = find_conda() python_version = self._get_python_env_version() + python = f"python={python_version}" + + cache_conf = self.python_cache() + + if "conda_env" in self.conf: + env_path = Path(self.conf["conda_env"]) + # conda env create does not have a --channel argument nor does it take + # dependencies specifications (e.g., python=3.8). These must all be specified + # in the conda-env.yml file + yaml = YAML() + env_file = yaml.load(env_path) + env_file["dependencies"].append(python) + + tmp_env = tempfile.NamedTemporaryFile( + dir=env_path.parent, + prefix="tox_conda_tmp", + suffix=".yaml", + delete=False, + ) + yaml.dump(env_file, tmp_env) + tmp_env.close() - conf = self.python_cache() + cmd = f"'{conda_exe}' create {cache_conf['conda_env_spec']} '{cache_conf['conda_env']}' --yes --quiet --file {tmp_env.name}" + tear_down = lambda: Path(tmp_env.name).unlink() + else: + cmd = f"'{conda_exe}' create {cache_conf['conda_env_spec']} '{cache_conf['conda_env']}' {python} --yes --quiet" - cmd = f"'{conda_exe}' create {conf['conda_env_spec']} '{conf['conda_env']}' python={python_version} --yes --quiet" try: cmd_list = shlex.split(cmd) subprocess.run(cmd_list, check=True) @@ -86,6 +109,9 @@ def create_python_env(self) -> None: raise Fail( f"Failed to create '{self.env_dir}' conda environment. Error: {e}" ) + finally: + if tear_down: + tear_down() def python_cache(self) -> Dict[str, Any]: base = super().python_cache() @@ -108,7 +134,7 @@ def python_cache(self) -> Dict[str, Any]: @property def executor(self) -> Execute: - def create_conda_command_prefix(): + def get_conda_command_prefix(): conf = self.python_cache() return [ "conda", @@ -126,7 +152,7 @@ def build_instance( out: SyncWrite, err: SyncWrite, ) -> ExecuteInstance: - conda_cmd = create_conda_command_prefix() + conda_cmd = get_conda_command_prefix() conda_request = ExecuteRequest( conda_cmd + request.cmd, @@ -258,6 +284,13 @@ def postprocess_path_option(testenv_config, value): @impl def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: + env_conf.add_config( + "conda_name", + of_type=str, + desc="Specifies the name of the conda environment. By default, .tox/ is used.", + default=None + ) + env_conf.add_config( "conda_env", of_type="path", @@ -265,6 +298,7 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: default=None, post_process=postprocess_path_option, ) + env_conf.add_config( "conda_spec", of_type="path", From c7929808a5b00d1f0cdcb5d0772d6f65d3331031 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Mon, 20 Feb 2023 21:12:22 +0000 Subject: [PATCH 05/39] Handle conda_env. --- tox_conda/plugin.py | 48 ++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 0f5b8eb..9d8e049 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -6,6 +6,7 @@ import tempfile from pathlib import Path from functools import partial +import hashlib MISSING_CONDA_ERROR = "Cannot locate the conda executable." @@ -42,6 +43,8 @@ from tox.tox_env.python.virtual_env.runner import VirtualEnvRunner from tox.tox_env.python.pip.req_file import PythonDeps +from ruamel.yaml import YAML + __all__ = [] @@ -79,8 +82,8 @@ def create_python_env(self) -> None: cache_conf = self.python_cache() - if "conda_env" in self.conf: - env_path = Path(self.conf["conda_env"]) + if self.conf["conda_env"]: + env_path = Path(self.conf["conda_env"]).resolve() # conda env create does not have a --channel argument nor does it take # dependencies specifications (e.g., python=3.8). These must all be specified # in the conda-env.yml file @@ -88,20 +91,20 @@ def create_python_env(self) -> None: env_file = yaml.load(env_path) env_file["dependencies"].append(python) - tmp_env = tempfile.NamedTemporaryFile( + tmp_env_file = tempfile.NamedTemporaryFile( dir=env_path.parent, prefix="tox_conda_tmp", suffix=".yaml", delete=False, ) - yaml.dump(env_file, tmp_env) - tmp_env.close() + yaml.dump(env_file, tmp_env_file) + tmp_env_file.close() - cmd = f"'{conda_exe}' create {cache_conf['conda_env_spec']} '{cache_conf['conda_env']}' --yes --quiet --file {tmp_env.name}" - tear_down = lambda: Path(tmp_env.name).unlink() + cmd = f"'{conda_exe}' env create --file '{tmp_env_file.name}' --quiet --force" + tear_down = lambda: Path(tmp_env_file.name).unlink() else: cmd = f"'{conda_exe}' create {cache_conf['conda_env_spec']} '{cache_conf['conda_env']}' {python} --yes --quiet" - + tear_down = lambda: None try: cmd_list = shlex.split(cmd) subprocess.run(cmd_list, check=True) @@ -110,25 +113,32 @@ def create_python_env(self) -> None: f"Failed to create '{self.env_dir}' conda environment. Error: {e}" ) finally: - if tear_down: - tear_down() + tear_down() def python_cache(self) -> Dict[str, Any]: base = super().python_cache() conda_name = getattr(self.options, "conda_name", None) - if not conda_name and "conda_name" in self.conf: + if not conda_name: conda_name = self.conf["conda_name"] if conda_name: conda_env_spec = "-n" conda_env = conda_name + conda_hash = "" + elif self.conf["conda_env"]: + conda_env_spec = f"-n" + env_path = Path(self.conf["conda_env"]).resolve() + env_file = YAML().load(env_path) + conda_env = env_file["name"] + conda_hash = hash_file(Path(self.conf["conda_env"]).resolve()) else: conda_env_spec = "-p" conda_env = f"{self.env_dir}" + conda_hash = "" base.update( - {"conda_env_spec": conda_env_spec, "conda_env": conda_env}, + {"conda_env_spec": conda_env_spec, "conda_env": conda_env, "conda_hash": conda_hash}, ) return base @@ -276,7 +286,7 @@ def tox_register_tox_env(register: ToxEnvRegister) -> None: # noqa: U100 pass -def postprocess_path_option(testenv_config, value): +def postprocess_path_option(*args, **kwargs): if value == testenv_config.config.toxinidir: return None return value @@ -293,12 +303,12 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: env_conf.add_config( "conda_env", - of_type="path", + of_type=str, desc="specify a conda environment.yml file", default=None, - post_process=postprocess_path_option, + # post_process=postprocess_path_option, ) - + env_conf.add_config( "conda_spec", of_type="path", @@ -383,6 +393,12 @@ def find_conda() -> Path: raise Fail("Failed to find 'conda' executable.") +def hash_file(file: Path) -> str: + with open(file.name, 'rb') as f: + sha1 = hashlib.sha1() + sha1.update(f.read()) + return sha1.hexdigest() + def _run_conda_process(args, venv, action, cwd): redirect = tox.reporter.verbosity() < tox.reporter.Verbosity.DEBUG venv._pcall(args, venv=False, action=action, cwd=cwd, redirect=redirect) From 838d5e14607713e7142e4b2bfff22c38771bca23 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Tue, 21 Feb 2023 09:36:36 +0000 Subject: [PATCH 06/39] Remove unused code. --- tox_conda/plugin.py | 169 -------------------------------------------- 1 file changed, 169 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 9d8e049..e30cf4a 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -7,11 +7,6 @@ from pathlib import Path from functools import partial import hashlib - - -MISSING_CONDA_ERROR = "Cannot locate the conda executable." - - import json import os import re @@ -286,12 +281,6 @@ def tox_register_tox_env(register: ToxEnvRegister) -> None: # noqa: U100 pass -def postprocess_path_option(*args, **kwargs): - if value == testenv_config.config.toxinidir: - return None - return value - - @impl def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: env_conf.add_config( @@ -306,7 +295,6 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: of_type=str, desc="specify a conda environment.yml file", default=None, - # post_process=postprocess_path_option, ) env_conf.add_config( @@ -314,7 +302,6 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: of_type="path", desc="specify a conda spec-file.txt file", default=None, - post_process=postprocess_path_option, ) root = env_conf._conf.core["tox_root"] @@ -343,33 +330,6 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: ) -# @hookimpl -def tox_configure(config): - # This is a pretty cheesy workaround. It allows tox to consider changes to - # the conda dependencies when it decides whether an existing environment - # needs to be updated before being used. - - # Set path to the conda executable because it cannot be determined once - # an env has already been created. - conda_exe = find_conda() - - for envconfig in config.envconfigs.values(): - # Make sure the right environment is activated. This works because we're - # creating environments using the `-p/--prefix` option in `tox_testenv_create` - envconfig.setenv["CONDA_DEFAULT_ENV"] = envconfig.setenv["TOX_ENV_DIR"] - - conda_deps = [DepConfig(str(name)) for name in envconfig.conda_deps] - # Append filenames of additional dependency sources. tox will automatically hash - # their contents to detect changes. - if envconfig.conda_spec is not None: - conda_deps.append(DepConfig(envconfig.conda_spec)) - if envconfig.conda_env is not None: - conda_deps.append(DepConfig(envconfig.conda_env)) - envconfig.deps.extend(conda_deps) - - envconfig.conda_exe = conda_exe - - def find_conda() -> Path: # This should work if we're not already in an environment conda_exe = os.environ.get("_CONDA_EXE") @@ -399,76 +359,6 @@ def hash_file(file: Path) -> str: sha1.update(f.read()) return sha1.hexdigest() -def _run_conda_process(args, venv, action, cwd): - redirect = tox.reporter.verbosity() < tox.reporter.Verbosity.DEBUG - venv._pcall(args, venv=False, action=action, cwd=cwd, redirect=redirect) - - -# @hookimpl -def tox_testenv_create(venv, action): - tox.venv.cleanup_for_venv(venv) - basepath = venv.path.dirpath() - - # Check for venv.envconfig.sitepackages and venv.config.alwayscopy here - envdir = venv.envconfig.envdir - python = get_py_version(venv.envconfig, action) - - if venv.envconfig.conda_env is not None: - env_path = Path(venv.envconfig.conda_env) - # conda env create does not have a --channel argument nor does it take - # dependencies specifications (e.g., python=3.8). These must all be specified - # in the conda-env.yml file - yaml = YAML() - env_file = yaml.load(env_path) - env_file["dependencies"].append(python) - - tmp_env = tempfile.NamedTemporaryFile( - dir=env_path.parent, - prefix="tox_conda_tmp", - suffix=".yaml", - delete=False, - ) - yaml.dump(env_file, tmp_env) - - args = [ - venv.envconfig.conda_exe, - "env", - "create", - "-p", - envdir, - "--file", - tmp_env.name, - ] - tmp_env.close() - _run_conda_process(args, venv, action, basepath) - Path(tmp_env.name).unlink() - - else: - args = [venv.envconfig.conda_exe, "create", "--yes", "-p", envdir] - for channel in venv.envconfig.conda_channels: - args += ["--channel", channel] - - # Add end-user conda create args - args += venv.envconfig.conda_create_args - - args += [python] - - _run_conda_process(args, venv, action, basepath) - - venv.envconfig.conda_python = python - - # let the venv know about the target interpreter just installed in our conda env, otherwise - # we'll have a mismatch later because tox expects the interpreter to be existing outside of - # the env - try: - del venv.envconfig.config.interpreters.name2executable[venv.name] - except KeyError: - pass - - venv.envconfig.config.interpreters.get_executable(venv.envconfig) - - return True - def install_conda_deps(venv, action, basepath, envdir): # Account for the fact that we have a list of DepOptions @@ -525,62 +415,3 @@ def tox_testenv_install_deps(venv, action): venv.envconfig.deps = saved_deps return True - - -# @hookimpl -def tox_get_python_executable(envconfig): - if tox.INFO.IS_WIN: - path = envconfig.envdir.join("python.exe") - else: - path = envconfig.envdir.join("bin", "python") - if path.exists(): - return path - - -# Monkey patch TestenConfig get_envpython to fix tox behavior with tox-conda under windows -def get_envpython(self): - """Override get_envpython to handle windows where the interpreter in at the env root dir.""" - original_envpython = self.__get_envpython() - if original_envpython.exists(): - return original_envpython - if tox.INFO.IS_WIN: - return self.envdir.join("python") - - -# TestenvConfig.__get_envpython = TestenvConfig.get_envpython -# TestenvConfig.get_envpython = get_envpython - - -# Monkey patch TestenvConfig _venv_lookup to fix tox behavior with tox-conda under windows -def venv_lookup(self, name): - """Override venv_lookup to also look at the env root dir under windows.""" - paths = [self.envconfig.envbindir] - # In Conda environments on Windows, the Python executable is installed in - # the top-level environment directory, as opposed to virtualenvs, where it - # is installed in the Scripts directory. Tox assumes that looking in the - # Scripts directory is sufficient, which is why this workaround is required. - if tox.INFO.IS_WIN: - paths += [self.envconfig.envdir] - return py.path.local.sysfind(name, paths=paths) - - -# VirtualEnv._venv_lookup = venv_lookup - - -# @hookimpl(hookwrapper=True) -def tox_runtest_pre(venv): - with activate_env(venv): - yield - - -# @hookimpl -def tox_runtest(venv, redirect): - with activate_env(venv): - tox.venv.tox_runtest(venv, redirect) - return True - - -# @hookimpl(hookwrapper=True) -def tox_runtest_post(venv): - with activate_env(venv): - yield From 7a0c51e02ea0fb39598ef78566670f08bf840760 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Tue, 21 Feb 2023 17:49:18 +0000 Subject: [PATCH 07/39] Installation. --- tox_conda/plugin.py | 248 +++++++++++++++++++++++--------------------- 1 file changed, 130 insertions(+), 118 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index e30cf4a..ba5f466 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -70,39 +70,67 @@ def _get_python_env_version(self): else: return self.base_python.version_dot + def python_cache(self) -> Dict[str, Any]: + conda_dict = {} + + conda_name = getattr(self.options, "conda_name", None) + if not conda_name: + conda_name = self.conf["conda_name"] + + if conda_name: + conda_dict["env_spec"] = "-n" + conda_dict["env"] = conda_name + elif self.conf["conda_env"]: + conda_dict["env_spec"] = "-n" + env_path = Path(self.conf["conda_env"]).resolve() + env_file = YAML().load(env_path) + conda_dict["env"] = env_file["name"] + conda_dict["env_path"] = str(env_path) + conda_dict["env_hash"] = hash_file(Path(self.conf["conda_env"]).resolve()) + else: + conda_dict["env_spec"] = "-p" + conda_dict["env"] = str(self.env_dir) + + _, conda_deps = self.conf["conda_deps"].unroll() + if conda_deps: + conda_dict["deps"] = conda_deps + + conda_spec = self.conf["conda_spec"] + if conda_spec: + conda_dict["spec"] = conda_spec + conda_dict["spec_hash"] = hash_file(Path(conda_spec).resolve()) + + conda_channels = self.conf["conda_channels"] + if conda_channels: + conda_dict["channels"] = conda_channels + + conda_install_args = self.conf["conda_install_args"] + if conda_install_args: + conda_dict["install_args"] = conda_install_args + + conda_create_args = self.conf["conda_create_args"] + if conda_create_args: + conda_dict["create_args"] = conda_create_args + + base = super().python_cache() + base.update( + {"conda": conda_dict}, + ) + return base + def create_python_env(self) -> None: conda_exe = find_conda() python_version = self._get_python_env_version() python = f"python={python_version}" - - cache_conf = self.python_cache() + conda_cache_conf = self.python_cache()["conda"] if self.conf["conda_env"]: - env_path = Path(self.conf["conda_env"]).resolve() - # conda env create does not have a --channel argument nor does it take - # dependencies specifications (e.g., python=3.8). These must all be specified - # in the conda-env.yml file - yaml = YAML() - env_file = yaml.load(env_path) - env_file["dependencies"].append(python) - - tmp_env_file = tempfile.NamedTemporaryFile( - dir=env_path.parent, - prefix="tox_conda_tmp", - suffix=".yaml", - delete=False, - ) - yaml.dump(env_file, tmp_env_file) - tmp_env_file.close() - - cmd = f"'{conda_exe}' env create --file '{tmp_env_file.name}' --quiet --force" - tear_down = lambda: Path(tmp_env_file.name).unlink() + create_command, tear_down = CondaEnvRunner._generate_env_create_command(conda_exe, python, conda_cache_conf) else: - cmd = f"'{conda_exe}' create {cache_conf['conda_env_spec']} '{cache_conf['conda_env']}' {python} --yes --quiet" - tear_down = lambda: None + create_command, tear_down = CondaEnvRunner._generate_create_command(conda_exe, python, conda_cache_conf) try: - cmd_list = shlex.split(cmd) - subprocess.run(cmd_list, check=True) + create_command_args = shlex.split(create_command) + subprocess.run(create_command_args, check=True) except subprocess.CalledProcessError as e: raise Fail( f"Failed to create '{self.env_dir}' conda environment. Error: {e}" @@ -110,44 +138,86 @@ def create_python_env(self) -> None: finally: tear_down() - def python_cache(self) -> Dict[str, Any]: - base = super().python_cache() + install_command = CondaEnvRunner._generate_install_command(conda_exe, python, conda_cache_conf) + if install_command: + try: + install_command_args = shlex.split(install_command) + subprocess.run(install_command_args, check=True) + except subprocess.CalledProcessError as e: + raise Fail( + f"Failed to install dependencies in conda environment. Error: {e}" + ) + - conda_name = getattr(self.options, "conda_name", None) - if not conda_name: - conda_name = self.conf["conda_name"] + @staticmethod + def _generate_env_create_command(conda_exe, python, conda_cache_conf): + env_path = Path(conda_cache_conf["env_path"]).resolve() + # conda env create does not have a --channel argument nor does it take + # dependencies specifications (e.g., python=3.8). These must all be specified + # in the conda-env.yml file + yaml = YAML() + env_file = yaml.load(env_path) + env_file["dependencies"].append(python) + + tmp_env_file = tempfile.NamedTemporaryFile( + dir=env_path.parent, + prefix="tox_conda_tmp", + suffix=".yaml", + delete=False, + ) + yaml.dump(env_file, tmp_env_file) + tmp_env_file.close() - if conda_name: - conda_env_spec = "-n" - conda_env = conda_name - conda_hash = "" - elif self.conf["conda_env"]: - conda_env_spec = f"-n" - env_path = Path(self.conf["conda_env"]).resolve() - env_file = YAML().load(env_path) - conda_env = env_file["name"] - conda_hash = hash_file(Path(self.conf["conda_env"]).resolve()) - else: - conda_env_spec = "-p" - conda_env = f"{self.env_dir}" - conda_hash = "" + cmd = f"'{conda_exe}' env create --file '{tmp_env_file.name}' --quiet --force" + tear_down = lambda: Path(tmp_env_file.name).unlink() - base.update( - {"conda_env_spec": conda_env_spec, "conda_env": conda_env, "conda_hash": conda_hash}, - ) - return base + return cmd, tear_down + + @staticmethod + def _generate_create_command(conda_exe, python, conda_cache_conf): + cmd = f"'{conda_exe}' create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}' {python} --yes --quiet" + for arg in conda_cache_conf.get("create_args", []): + cmd += f" '{arg}'" + + tear_down = lambda: None + return cmd, tear_down + + @staticmethod + def _generate_install_command(conda_exe, python, conda_cache_conf): + # Check if there is anything to install + if "deps" not in conda_cache_conf and "spec" not in conda_cache_conf: + return None + + cmd = f"'{conda_exe}' install --quiet --yes {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}'" + for channel in conda_cache_conf.get("channels", []): + cmd += f" --channel {channel}" + + # Add end-user conda install args + for arg in conda_cache_conf.get("install_args", []): + cmd += f" {arg}" + + # We include the python version in the conda requirements in order to make + # sure that none of the other conda requirements inadvertently downgrade + # python in this environment. If any of the requirements are in conflict + # with the installed python version, installation will fail (which is what + # we want). + cmd += f" {python}" + + for dep in conda_cache_conf.get("deps", []): + cmd += f" {dep}" + + if "spec" in conda_cache_conf: + cmd += f" --file={conda_cache_conf['spec']}" + + return cmd @property def executor(self) -> Execute: def get_conda_command_prefix(): - conf = self.python_cache() - return [ - "conda", - "run", - conf["conda_env_spec"], - conf["conda_env"], - "--live-stream", - ] + conda_exe = find_conda() + cache_conf = self.python_cache() + cmd = f"'{conda_exe}' run {cache_conf['conda']['env_spec']} '{cache_conf['conda']['env']}' --live-stream" + return shlex.split(cmd) class CondaExecutor(LocalSubProcessExecutor): def build_instance( @@ -299,7 +369,7 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: env_conf.add_config( "conda_spec", - of_type="path", + of_type=str, desc="specify a conda spec-file.txt file", default=None, ) @@ -314,22 +384,21 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: ) env_conf.add_config( - "conda_channels", of_type="line-list", desc="each line specifies a conda channel", default=None, + "conda_channels", of_type=List[str], desc="each line specifies a conda channel", default=None, ) env_conf.add_config( "conda_install_args", - of_type="line-list", + of_type=List[str], desc="each line specifies a conda install argument",default=None, ) env_conf.add_config( "conda_create_args", - of_type="line-list", + of_type=List[str], desc="each line specifies a conda create argument",default=None, ) - def find_conda() -> Path: # This should work if we're not already in an environment conda_exe = os.environ.get("_CONDA_EXE") @@ -358,60 +427,3 @@ def hash_file(file: Path) -> str: sha1 = hashlib.sha1() sha1.update(f.read()) return sha1.hexdigest() - - -def install_conda_deps(venv, action, basepath, envdir): - # Account for the fact that we have a list of DepOptions - conda_deps = [str(dep.name) for dep in venv.envconfig.conda_deps] - # Add the conda-spec.txt file to the end of the conda deps b/c any deps - # after --file option(s) are ignored - if venv.envconfig.conda_spec is not None: - conda_deps.append("--file={}".format(venv.envconfig.conda_spec)) - - action.setactivity("installcondadeps", ", ".join(conda_deps)) - - # Install quietly to make the log cleaner - args = [venv.envconfig.conda_exe, "install", "--quiet", "--yes", "-p", envdir] - for channel in venv.envconfig.conda_channels: - args += ["--channel", channel] - - # Add end-user conda install args - args += venv.envconfig.conda_install_args - - # We include the python version in the conda requirements in order to make - # sure that none of the other conda requirements inadvertently downgrade - # python in this environment. If any of the requirements are in conflict - # with the installed python version, installation will fail (which is what - # we want). - args += [venv.envconfig.conda_python] + conda_deps - - _run_conda_process(args, venv, action, basepath) - - -# @hookimpl -def tox_testenv_install_deps(venv, action): - # Save the deps before we make temporary changes. - saved_deps = copy.deepcopy(venv.envconfig.deps) - - num_conda_deps = len(venv.envconfig.conda_deps) - if venv.envconfig.conda_spec is not None: - num_conda_deps += 1 - - if num_conda_deps > 0: - install_conda_deps(venv, action, venv.path.dirpath(), venv.envconfig.envdir) - - # Account for the fact that we added the conda_deps to the deps list in - # tox_configure (see comment there for rationale). We don't want them - # to be present when we call pip install. - if venv.envconfig.conda_env is not None: - num_conda_deps += 1 - if num_conda_deps > 0: - venv.envconfig.deps = venv.envconfig.deps[:-num_conda_deps] - - with activate_env(venv, action): - tox.venv.tox_testenv_install_deps(venv=venv, action=action) - - # Restore the deps. - venv.envconfig.deps = saved_deps - - return True From 93c897ac94057af963ff02726c0eee3321e37370 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Tue, 21 Feb 2023 17:54:05 +0000 Subject: [PATCH 08/39] Reformat. --- tox_conda/plugin.py | 79 ++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index ba5f466..205ed07 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -1,11 +1,4 @@ import copy -import os -import re -import shutil -import subprocess -import tempfile -from pathlib import Path -from functools import partial import hashlib import json import os @@ -13,36 +6,29 @@ import shlex import shutil import subprocess +import tempfile +from functools import partial from io import BytesIO, TextIOWrapper from pathlib import Path from time import sleep from typing import Any, Dict, List -from tox.execute.api import ( - Execute, - ExecuteInstance, - ExecuteOptions, - ExecuteRequest, - SyncWrite, -) -from tox.execute.local_sub_process import ( - LocalSubProcessExecuteInstance, - LocalSubProcessExecutor, -) +from ruamel.yaml import YAML + +from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteRequest, SyncWrite +from tox.execute.local_sub_process import LocalSubProcessExecuteInstance, LocalSubProcessExecutor from tox.plugin import impl -from tox.plugin.spec import EnvConfigSet, State, ToxEnvRegister, ToxParser -from tox.tox_env.api import StdinSource, ToxEnv, ToxEnvCreateArgs +from tox.plugin.spec import EnvConfigSet, State, ToxEnvRegister +from tox.tox_env.api import StdinSource, ToxEnvCreateArgs from tox.tox_env.errors import Fail from tox.tox_env.installer import Installer from tox.tox_env.python.pip.pip_install import Pip -from tox.tox_env.python.virtual_env.runner import VirtualEnvRunner from tox.tox_env.python.pip.req_file import PythonDeps - -from ruamel.yaml import YAML - +from tox.tox_env.python.virtual_env.runner import VirtualEnvRunner __all__ = [] + class CondaEnvRunner(VirtualEnvRunner): def __init__(self, create_args: ToxEnvCreateArgs) -> None: self._installer = None @@ -56,9 +42,7 @@ def id() -> str: # noqa A003 def _get_python_env_version(self): # Try to use base_python config - match = re.match( - r"python(\d)(?:\.(\d+))?(?:\.?(\d))?", self.conf["base_python"][0] - ) + match = re.match(r"python(\d)(?:\.(\d+))?(?:\.?(\d))?", self.conf["base_python"][0]) if match: groups = match.groups() version = groups[0] @@ -125,29 +109,30 @@ def create_python_env(self) -> None: conda_cache_conf = self.python_cache()["conda"] if self.conf["conda_env"]: - create_command, tear_down = CondaEnvRunner._generate_env_create_command(conda_exe, python, conda_cache_conf) + create_command, tear_down = CondaEnvRunner._generate_env_create_command( + conda_exe, python, conda_cache_conf + ) else: - create_command, tear_down = CondaEnvRunner._generate_create_command(conda_exe, python, conda_cache_conf) + create_command, tear_down = CondaEnvRunner._generate_create_command( + conda_exe, python, conda_cache_conf + ) try: create_command_args = shlex.split(create_command) subprocess.run(create_command_args, check=True) except subprocess.CalledProcessError as e: - raise Fail( - f"Failed to create '{self.env_dir}' conda environment. Error: {e}" - ) + raise Fail(f"Failed to create '{self.env_dir}' conda environment. Error: {e}") finally: tear_down() - install_command = CondaEnvRunner._generate_install_command(conda_exe, python, conda_cache_conf) + install_command = CondaEnvRunner._generate_install_command( + conda_exe, python, conda_cache_conf + ) if install_command: try: install_command_args = shlex.split(install_command) subprocess.run(install_command_args, check=True) except subprocess.CalledProcessError as e: - raise Fail( - f"Failed to install dependencies in conda environment. Error: {e}" - ) - + raise Fail(f"Failed to install dependencies in conda environment. Error: {e}") @staticmethod def _generate_env_create_command(conda_exe, python, conda_cache_conf): @@ -202,7 +187,7 @@ def _generate_install_command(conda_exe, python, conda_cache_conf): # with the installed python version, installation will fail (which is what # we want). cmd += f" {python}" - + for dep in conda_cache_conf.get("deps", []): cmd += f" {dep}" @@ -257,7 +242,7 @@ def _default_pass_env(self) -> List[str]: env = super()._default_pass_env() env.append("*CONDA*") return env - + def env_site_package_dir(self) -> Path: """The site package folder within the tox environment.""" cmd = 'from sysconfig import get_paths; print(get_paths()["purelib"])' @@ -357,7 +342,7 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: "conda_name", of_type=str, desc="Specifies the name of the conda environment. By default, .tox/ is used.", - default=None + default=None, ) env_conf.add_config( @@ -384,21 +369,27 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: ) env_conf.add_config( - "conda_channels", of_type=List[str], desc="each line specifies a conda channel", default=None, + "conda_channels", + of_type=List[str], + desc="each line specifies a conda channel", + default=None, ) env_conf.add_config( "conda_install_args", of_type=List[str], - desc="each line specifies a conda install argument",default=None, + desc="each line specifies a conda install argument", + default=None, ) env_conf.add_config( "conda_create_args", of_type=List[str], - desc="each line specifies a conda create argument",default=None, + desc="each line specifies a conda create argument", + default=None, ) + def find_conda() -> Path: # This should work if we're not already in an environment conda_exe = os.environ.get("_CONDA_EXE") @@ -423,7 +414,7 @@ def find_conda() -> Path: def hash_file(file: Path) -> str: - with open(file.name, 'rb') as f: + with open(file.name, "rb") as f: sha1 = hashlib.sha1() sha1.update(f.read()) return sha1.hexdigest() From 75047263750d5cdc929f1c85e7d0e52b46aa8e67 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:56:55 +0000 Subject: [PATCH 09/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tox_conda/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 205ed07..aaa0395 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -14,7 +14,6 @@ from typing import Any, Dict, List from ruamel.yaml import YAML - from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteRequest, SyncWrite from tox.execute.local_sub_process import LocalSubProcessExecuteInstance, LocalSubProcessExecutor from tox.plugin import impl From dfb59dc63fdcfcadf3864cd03e2952eb8efe0ce8 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Mon, 24 Apr 2023 12:13:19 +0100 Subject: [PATCH 10/39] Inherit from PythonRun and add type info. --- tox_conda/plugin.py | 47 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index aaa0395..1782446 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -1,4 +1,3 @@ -import copy import hashlib import json import os @@ -6,6 +5,7 @@ import shlex import shutil import subprocess +import sys import tempfile from functools import partial from io import BytesIO, TextIOWrapper @@ -21,14 +21,16 @@ from tox.tox_env.api import StdinSource, ToxEnvCreateArgs from tox.tox_env.errors import Fail from tox.tox_env.installer import Installer +from tox.tox_env.python.api import PythonInfo, VersionInfo +from tox.tox_env.python.runner import PythonRun from tox.tox_env.python.pip.pip_install import Pip from tox.tox_env.python.pip.req_file import PythonDeps -from tox.tox_env.python.virtual_env.runner import VirtualEnvRunner + __all__ = [] -class CondaEnvRunner(VirtualEnvRunner): +class CondaEnvRunner(PythonRun): def __init__(self, create_args: ToxEnvCreateArgs) -> None: self._installer = None self._executor = None @@ -38,6 +40,35 @@ def __init__(self, create_args: ToxEnvCreateArgs) -> None: @staticmethod def id() -> str: # noqa A003 return "conda" + + def _get_python(self, base_python: List[str]) -> PythonInfo | None: + exe_path = base_python[0] + + output = subprocess.check_output([exe_path, "-c", "import platform, sys; print(platform.python_implementation());print(platform.sys.version_info);print(sys.version);print(sys.maxsize > 2**32);print(platform.system())"]) + output = output.decode("utf-8").strip().split(os.linesep) + + implementation, version_info, version, is_64, platform_name = output + + is_64 = bool(is_64) + match = re.match(r"sys\.version_info\(major=(\d+), minor=(\d+), micro=(\d+), releaselevel='(\w+)', serial=(\d+)\)", version_info) + version_info = VersionInfo( + major=int(match.group(1)), + minor=int(match.group(2)), + micro=int(match.group(3)), + releaselevel=match.group(4), + serial=int(match.group(5)) + ) + extra = {"executable_path": exe_path} + + return PythonInfo(implementation, version_info, version, is_64, platform_name, extra) + + @property + def _package_tox_env_type(self) -> str: + raise NotImplementedError + + @property + def _external_pkg_tox_env_type(self) -> str: + raise NotImplementedError def _get_python_env_version(self): # Try to use base_python config @@ -53,6 +84,10 @@ def _get_python_env_version(self): else: return self.base_python.version_dot + @property + def runs_on_platform(self) -> str: + return sys.platform + def python_cache(self) -> Dict[str, Any]: conda_dict = {} @@ -134,7 +169,7 @@ def create_python_env(self) -> None: raise Fail(f"Failed to install dependencies in conda environment. Error: {e}") @staticmethod - def _generate_env_create_command(conda_exe, python, conda_cache_conf): + def _generate_env_create_command(conda_exe: Path, python: str, conda_cache_conf: Dict[str, str]): env_path = Path(conda_cache_conf["env_path"]).resolve() # conda env create does not have a --channel argument nor does it take # dependencies specifications (e.g., python=3.8). These must all be specified @@ -158,7 +193,7 @@ def _generate_env_create_command(conda_exe, python, conda_cache_conf): return cmd, tear_down @staticmethod - def _generate_create_command(conda_exe, python, conda_cache_conf): + def _generate_create_command(conda_exe: Path, python: str, conda_cache_conf: Dict[str, str]): cmd = f"'{conda_exe}' create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}' {python} --yes --quiet" for arg in conda_cache_conf.get("create_args", []): cmd += f" '{arg}'" @@ -167,7 +202,7 @@ def _generate_create_command(conda_exe, python, conda_cache_conf): return cmd, tear_down @staticmethod - def _generate_install_command(conda_exe, python, conda_cache_conf): + def _generate_install_command(conda_exe: Path, python: str, conda_cache_conf: Dict[str, str]): # Check if there is anything to install if "deps" not in conda_cache_conf and "spec" not in conda_cache_conf: return None From 4aed14cdd0ecedccabd9ab6677a76614ae404e71 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Mon, 24 Apr 2023 12:15:03 +0100 Subject: [PATCH 11/39] Reformat plugin with isort and black. --- tox_conda/plugin.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 1782446..88c0a5c 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -22,10 +22,9 @@ from tox.tox_env.errors import Fail from tox.tox_env.installer import Installer from tox.tox_env.python.api import PythonInfo, VersionInfo -from tox.tox_env.python.runner import PythonRun from tox.tox_env.python.pip.pip_install import Pip from tox.tox_env.python.pip.req_file import PythonDeps - +from tox.tox_env.python.runner import PythonRun __all__ = [] @@ -40,26 +39,35 @@ def __init__(self, create_args: ToxEnvCreateArgs) -> None: @staticmethod def id() -> str: # noqa A003 return "conda" - + def _get_python(self, base_python: List[str]) -> PythonInfo | None: exe_path = base_python[0] - - output = subprocess.check_output([exe_path, "-c", "import platform, sys; print(platform.python_implementation());print(platform.sys.version_info);print(sys.version);print(sys.maxsize > 2**32);print(platform.system())"]) + + output = subprocess.check_output( + [ + exe_path, + "-c", + "import platform, sys; print(platform.python_implementation());print(platform.sys.version_info);print(sys.version);print(sys.maxsize > 2**32);print(platform.system())", + ] + ) output = output.decode("utf-8").strip().split(os.linesep) - + implementation, version_info, version, is_64, platform_name = output - + is_64 = bool(is_64) - match = re.match(r"sys\.version_info\(major=(\d+), minor=(\d+), micro=(\d+), releaselevel='(\w+)', serial=(\d+)\)", version_info) + match = re.match( + r"sys\.version_info\(major=(\d+), minor=(\d+), micro=(\d+), releaselevel='(\w+)', serial=(\d+)\)", + version_info, + ) version_info = VersionInfo( major=int(match.group(1)), minor=int(match.group(2)), micro=int(match.group(3)), releaselevel=match.group(4), - serial=int(match.group(5)) + serial=int(match.group(5)), ) extra = {"executable_path": exe_path} - + return PythonInfo(implementation, version_info, version, is_64, platform_name, extra) @property @@ -169,7 +177,9 @@ def create_python_env(self) -> None: raise Fail(f"Failed to install dependencies in conda environment. Error: {e}") @staticmethod - def _generate_env_create_command(conda_exe: Path, python: str, conda_cache_conf: Dict[str, str]): + def _generate_env_create_command( + conda_exe: Path, python: str, conda_cache_conf: Dict[str, str] + ): env_path = Path(conda_cache_conf["env_path"]).resolve() # conda env create does not have a --channel argument nor does it take # dependencies specifications (e.g., python=3.8). These must all be specified From 6afdb367af51c070cbf8f5ff37c223e5ef572ca0 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Tue, 30 May 2023 14:10:25 +0100 Subject: [PATCH 12/39] WIP. --- setup.cfg | 2 +- tox.ini | 6 +++--- tox_conda/plugin.py | 39 +++++++++++++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/setup.cfg b/setup.cfg index b8a9bb3..175c186 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,7 @@ classifiers = packages = find: install_requires = ruamel.yaml>=0.15.0,<0.18 - tox>=4 + tox>=4,<5 python_requires = >=3.5 [options.packages.find] diff --git a/tox.ini b/tox.ini index 16db4ba..b445310 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ envlist = pkg_meta isolated_build = true skip_missing_interpreters = true -minversion = 3.14.0 +minversion = 4 [testenv] description = run test suite under {basepython} @@ -47,7 +47,7 @@ commands = [testenv:coverage] description = [run locally after tests]: combine coverage data and create report; - generates a diff coverage against origin/master (can be changed by setting DIFF_AGAINST env var) + generates a diff coverage against origin/main (can be changed by setting DIFF_AGAINST env var) passenv = DIFF_AGAINST setenv = @@ -63,7 +63,7 @@ commands = coverage report -m coverage xml -o {toxworkdir}/coverage.xml coverage html -d {toxworkdir}/htmlcov - diff-cover --compare-branch {env:DIFF_AGAINST:origin/master} {toxworkdir}/coverage.xml + diff-cover --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}/coverage.xml depends = py39 py38 diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 88c0a5c..6dda538 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -47,7 +47,13 @@ def _get_python(self, base_python: List[str]) -> PythonInfo | None: [ exe_path, "-c", - "import platform, sys; print(platform.python_implementation());print(platform.sys.version_info);print(sys.version);print(sys.maxsize > 2**32);print(platform.system())", + ( + "import platform, sys;" + "print(platform.python_implementation());" + "print(platform.sys.version_info);" + "print(sys.version);" + "print(sys.maxsize > 2**32);print(platform.system())" + ), ] ) output = output.decode("utf-8").strip().split(os.linesep) @@ -56,7 +62,10 @@ def _get_python(self, base_python: List[str]) -> PythonInfo | None: is_64 = bool(is_64) match = re.match( - r"sys\.version_info\(major=(\d+), minor=(\d+), micro=(\d+), releaselevel='(\w+)', serial=(\d+)\)", + ( + r"sys\.version_info\(major=(\d+), minor=(\d+), micro=(\d+), releaselevel='(\w+)'," + r" serial=(\d+)\)" + ), version_info, ) version_info = VersionInfo( @@ -198,17 +207,24 @@ def _generate_env_create_command( tmp_env_file.close() cmd = f"'{conda_exe}' env create --file '{tmp_env_file.name}' --quiet --force" - tear_down = lambda: Path(tmp_env_file.name).unlink() + + def tear_down(): + return Path(tmp_env_file.name).unlink() return cmd, tear_down @staticmethod def _generate_create_command(conda_exe: Path, python: str, conda_cache_conf: Dict[str, str]): - cmd = f"'{conda_exe}' create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}' {python} --yes --quiet" + cmd = ( + f"'{conda_exe}' create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}'" + f" {python} --yes --quiet" + ) for arg in conda_cache_conf.get("create_args", []): cmd += f" '{arg}'" - tear_down = lambda: None + def tear_down(): + return None + return cmd, tear_down @staticmethod @@ -217,7 +233,10 @@ def _generate_install_command(conda_exe: Path, python: str, conda_cache_conf: Di if "deps" not in conda_cache_conf and "spec" not in conda_cache_conf: return None - cmd = f"'{conda_exe}' install --quiet --yes {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}'" + cmd = ( + f"'{conda_exe}' install --quiet --yes" + f" {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}'" + ) for channel in conda_cache_conf.get("channels", []): cmd += f" --channel {channel}" @@ -245,7 +264,10 @@ def executor(self) -> Execute: def get_conda_command_prefix(): conda_exe = find_conda() cache_conf = self.python_cache() - cmd = f"'{conda_exe}' run {cache_conf['conda']['env_spec']} '{cache_conf['conda']['env']}' --live-stream" + cmd = ( + f"'{conda_exe}' run" + f" {cache_conf['conda']['env_spec']} '{cache_conf['conda']['env']}' --live-stream" + ) return shlex.split(cmd) class CondaExecutor(LocalSubProcessExecutor): @@ -364,7 +386,8 @@ def _ensure_python_env_exists(self) -> None: self._created = True else: raise Fail( - f"{self.env_dir} already exists, but it is not a conda environment. Delete in manually first." + f"{self.env_dir} already exists, but it is not a conda environment. Delete in" + " manually first." ) From 1330cadc797d1107f633f22a3454786d03a194d3 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Wed, 31 May 2023 10:36:01 +0100 Subject: [PATCH 13/39] Fixes. --- tests/conftest.py | 2 +- tox_conda/plugin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8b36b3b..ee2f10c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1 @@ -from tox._pytestplugin import * # noqa +# from tox._pytestplugin import * # noqa diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 6dda538..1de61f5 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -11,7 +11,7 @@ from io import BytesIO, TextIOWrapper from pathlib import Path from time import sleep -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from ruamel.yaml import YAML from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteRequest, SyncWrite @@ -40,7 +40,7 @@ def __init__(self, create_args: ToxEnvCreateArgs) -> None: def id() -> str: # noqa A003 return "conda" - def _get_python(self, base_python: List[str]) -> PythonInfo | None: + def _get_python(self, base_python: List[str]) -> Optional[PythonInfo]: exe_path = base_python[0] output = subprocess.check_output( From 26fe70f4d7113c12b3f30a5d428f9a09df5772a7 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Wed, 31 May 2023 14:54:34 +0100 Subject: [PATCH 14/39] Plugin fixes. --- tox_conda/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 1de61f5..5a365bb 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -48,10 +48,10 @@ def _get_python(self, base_python: List[str]) -> Optional[PythonInfo]: exe_path, "-c", ( - "import platform, sys;" + "import os, platform, sys;" "print(platform.python_implementation());" "print(platform.sys.version_info);" - "print(sys.version);" + " print(sys.version.split(os.linesep)[0]);" "print(sys.maxsize > 2**32);print(platform.system())" ), ] @@ -81,11 +81,11 @@ def _get_python(self, base_python: List[str]) -> Optional[PythonInfo]: @property def _package_tox_env_type(self) -> str: - raise NotImplementedError + return "virtualenv-pep-517" @property def _external_pkg_tox_env_type(self) -> str: - raise NotImplementedError + return "virtualenv-cmd-builder" def _get_python_env_version(self): # Try to use base_python config From b51c6a7067732f0bc5fc1e534cc73cd20d0bb380 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Wed, 31 May 2023 15:06:04 +0100 Subject: [PATCH 15/39] Use executor instead of subprocess. --- tox_conda/plugin.py | 89 ++++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 5a365bb..219a8e0 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -33,6 +33,7 @@ class CondaEnvRunner(PythonRun): def __init__(self, create_args: ToxEnvCreateArgs) -> None: self._installer = None self._executor = None + self._external_executor = None self._created = False super().__init__(create_args) @@ -43,7 +44,7 @@ def id() -> str: # noqa A003 def _get_python(self, base_python: List[str]) -> Optional[PythonInfo]: exe_path = base_python[0] - output = subprocess.check_output( + output = self._run_pure( [ exe_path, "-c", @@ -54,9 +55,10 @@ def _get_python(self, base_python: List[str]) -> Optional[PythonInfo]: " print(sys.version.split(os.linesep)[0]);" "print(sys.maxsize > 2**32);print(platform.system())" ), - ] + ], + "_get_python", ) - output = output.decode("utf-8").strip().split(os.linesep) + output = output.split(os.linesep) implementation, version_info, version, is_64, platform_name = output @@ -169,9 +171,7 @@ def create_python_env(self) -> None: ) try: create_command_args = shlex.split(create_command) - subprocess.run(create_command_args, check=True) - except subprocess.CalledProcessError as e: - raise Fail(f"Failed to create '{self.env_dir}' conda environment. Error: {e}") + self._run_pure(create_command_args, "create_python_env-create") finally: tear_down() @@ -179,11 +179,8 @@ def create_python_env(self) -> None: conda_exe, python, conda_cache_conf ) if install_command: - try: - install_command_args = shlex.split(install_command) - subprocess.run(install_command_args, check=True) - except subprocess.CalledProcessError as e: - raise Fail(f"Failed to install dependencies in conda environment. Error: {e}") + install_command_args = shlex.split(install_command) + self._run_pure(install_command_args, "create_python_env-install") @staticmethod def _generate_env_create_command( @@ -259,6 +256,12 @@ def _generate_install_command(conda_exe: Path, python: str, conda_cache_conf: Di return cmd + @property + def external_executor(self) -> Execute: + if self._external_executor is None: + self._external_executor = LocalSubProcessExecutor(self.options.is_colored) + return self._external_executor + @property def executor(self) -> Execute: def get_conda_command_prefix(): @@ -311,27 +314,51 @@ def _default_pass_env(self) -> List[str]: def env_site_package_dir(self) -> Path: """The site package folder within the tox environment.""" - cmd = 'from sysconfig import get_paths; print(get_paths()["purelib"])' - path = self._call_python_in_conda_env(cmd, "env_site_package_dir") + script = 'from sysconfig import get_paths; print(get_paths()["purelib"])' + path = self._run_python_script_in_conda(script, "env_site_package_dir") return Path(path).resolve() def env_python(self) -> Path: """The python executable within the tox environment.""" - cmd = "import sys; print(sys.executable)" - path = self._call_python_in_conda_env(cmd, "env_python") + script = "import sys; print(sys.executable)" + path = self._run_in_conda(script, "env_python") return Path(path).resolve() def env_bin_dir(self) -> Path: """The binary folder within the tox environment.""" - cmd = 'from sysconfig import get_paths; print(get_paths()["scripts"])' - path = self._call_python_in_conda_env(cmd, "env_bin_dir") + script = 'from sysconfig import get_paths; print(get_paths()["scripts"])' + path = self._run_in_conda(script, "env_bin_dir") return Path(path).resolve() - def _call_python_in_conda_env(self, cmd: str, run_id: str): + def _run_python_script_in_conda(self, script: str, run_id: str): + cmd = "python -c".split() + [script] + return self._run_in_conda(cmd, run_id) + + def _run_in_conda(self, cmd: List[str], run_id: str): self._ensure_python_env_exists() - python_cmd = "python -c".split() + request = ExecuteRequest( + cmd, + self.conf["change_dir"], + self.environment_variables, + StdinSource.API, + run_id, + ) + + return self._run_with_executor(self.executor, request) + def _run_pure(self, cmd: List[str], run_id: str): + request = ExecuteRequest( + cmd, + self.conf["change_dir"], + self.environment_variables, + StdinSource.API, + run_id, + ) + + return self._run_with_executor(self.external_executor, request) + + def _run_with_executor(self, executor: Execute, request: ExecuteRequest): class NamedBytesIO(BytesIO): def __init__(self, name): self.name = name @@ -345,20 +372,13 @@ def __init__(self, name): out_err = out, err - request = ExecuteRequest( - python_cmd + [cmd], - self.conf["change_dir"], - self.environment_variables, - StdinSource.API, - run_id, - ) - - with self.executor.call(request, True, out_err, self) as execute_status: + with executor.call(request, True, out_err, self) as execute_status: while execute_status.wait() is None: sleep(0.01) if execute_status.exit_code != 0: raise Fail( - f"Failed to execute operation '{cmd}'. Stderr: {execute_status.err.decode()}" + f"Failed to execute operation '{request.cmd}'. " + f"Stderr: {execute_status.err.decode()}" ) return execute_status.out.decode().strip() @@ -374,14 +394,9 @@ def _ensure_python_env_exists(self) -> None: conda_exe = find_conda() cmd = f"'{conda_exe}' env list --json" - try: - cmd_list = shlex.split(cmd) - result: subprocess.CompletedProcess = subprocess.run( - cmd_list, check=True, capture_output=True - ) - except subprocess.CalledProcessError as e: - raise Fail(f"Failed to list conda environments. Error: {e}") - envs = json.loads(result.stdout.decode()) + cmd_list = shlex.split(cmd) + result = self._run_pure(cmd_list, "_ensure_python_env_exists") + envs = json.loads(result) if str(self.env_dir) in envs["envs"]: self._created = True else: From 52a8bfb758b70abefafd535760a7ab93a13b90e6 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Thu, 1 Jun 2023 18:00:48 +0100 Subject: [PATCH 16/39] Test infrastructure wip. --- tests/test_conda_env.py | 1091 +++++++++++++++++++++++---------------- tox_conda/plugin.py | 38 +- 2 files changed, 670 insertions(+), 459 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 1756846..4204a61 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -2,458 +2,647 @@ import os import pathlib import re +import subprocess +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Union, Sequence from unittest.mock import mock_open, patch + +from types import ModuleType, TracebackType + +import pytest import tox +from pytest import MonkeyPatch +from pytest_mock import MockerFixture from ruamel.yaml import YAML -from tox.venv import VirtualEnv - -from tox_conda.env_activator import PopenInActivatedEnv -from tox_conda.plugin import tox_testenv_create, tox_testenv_install_deps - - -def test_conda_create(newconfig, mocksession): - config = newconfig( - [], - """ - [testenv:py123] - """, - ) - - venv = VirtualEnv(config.envconfigs["py123"]) - assert venv.path == config.envconfigs["py123"].envdir - - with mocksession.newaction(venv.name, "getenv") as action: - tox_testenv_create(action=action, venv=venv) - pcalls = mocksession._pcalls - assert len(pcalls) >= 1 - call = pcalls[-1] - assert "conda" in call.args[0] - assert "create" == call.args[1] - assert "--yes" == call.args[2] - assert "-p" == call.args[3] - assert venv.path == call.args[4] - assert call.args[5].startswith("python=") - - -def create_test_env(config, mocksession, envname): - - venv = VirtualEnv(config.envconfigs[envname]) - with mocksession.newaction(venv.name, "getenv") as action: - tox_testenv_create(action=action, venv=venv) - pcalls = mocksession._pcalls - assert len(pcalls) >= 1 - pcalls[:] = [] - - return venv, action, pcalls - - -def test_install_deps_no_conda(newconfig, mocksession, monkeypatch): - """Test installation using conda when no conda_deps are given""" - # No longer remove the temporary script, so we can check its contents. - monkeypatch.delattr(PopenInActivatedEnv, "__del__", raising=False) - - env_name = "py123" - config = newconfig( - [], - """ - [testenv:{}] - deps= - numpy - -r requirements.txt - astropy - """.format( - env_name - ), - ) - - config.toxinidir.join("requirements.txt").write("") - - venv, action, pcalls = create_test_env(config, mocksession, env_name) - - assert len(venv.envconfig.deps) == 3 - assert len(venv.envconfig.conda_deps) == 0 - - tox_testenv_install_deps(action=action, venv=venv) - - assert len(pcalls) >= 1 - - call = pcalls[-1] - - if tox.INFO.IS_WIN: - script_lines = " ".join(call.args).split(" && ", maxsplit=1) - pattern = r"conda\.bat activate .*{}".format(re.escape(env_name)) - else: - # Get the cmd args from the script. - shell, cmd_script = call.args - assert shell == "/bin/sh" - with open(cmd_script) as stream: - script_lines = stream.readlines() - pattern = r"eval \"\$\(/.*/conda shell\.posix activate /.*/{}\)\"".format(env_name) - - assert re.match(pattern, script_lines[0]) - - cmd = script_lines[1].split() - assert cmd[-6:] == ["-m", "pip", "install", "numpy", "-rrequirements.txt", "astropy"] - - -def test_install_conda_deps(newconfig, mocksession): - config = newconfig( - [], - """ - [testenv:py123] - deps= - numpy - astropy - conda_deps= - pytest - asdf - """, - ) - - venv, action, pcalls = create_test_env(config, mocksession, "py123") - - assert len(venv.envconfig.conda_deps) == 2 - assert len(venv.envconfig.deps) == 2 + len(venv.envconfig.conda_deps) - - tox_testenv_install_deps(action=action, venv=venv) - # We expect two calls: one for conda deps, and one for pip deps - assert len(pcalls) >= 2 - call = pcalls[-2] - conda_cmd = call.args - assert "conda" in os.path.split(conda_cmd[0])[-1] - assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] - # Make sure that python is explicitly given as part of every conda install - # in order to avoid inadvertent upgrades of python itself. - assert conda_cmd[6].startswith("python=") - assert conda_cmd[7:9] == ["pytest", "asdf"] - - -def test_install_conda_no_pip(newconfig, mocksession): - config = newconfig( - [], - """ - [testenv:py123] - conda_deps= - pytest - asdf - """, - ) - - venv, action, pcalls = create_test_env(config, mocksession, "py123") - - assert len(venv.envconfig.conda_deps) == 2 - assert len(venv.envconfig.deps) == len(venv.envconfig.conda_deps) - - tox_testenv_install_deps(action=action, venv=venv) - # We expect only one call since there are no true pip dependencies - assert len(pcalls) >= 1 - - # Just a quick sanity check for the conda install command - call = pcalls[-1] - conda_cmd = call.args - assert "conda" in os.path.split(conda_cmd[0])[-1] - assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] - - -def test_update(tmpdir, newconfig, mocksession): - pkg = tmpdir.ensure("package.tar.gz") - config = newconfig( - [], - """ - [testenv:py123] - deps= - numpy - astropy - conda_deps= - pytest - asdf - """, - ) - - venv, action, pcalls = create_test_env(config, mocksession, "py123") - tox_testenv_install_deps(action=action, venv=venv) - - venv.hook.tox_testenv_create = tox_testenv_create - venv.hook.tox_testenv_install_deps = tox_testenv_install_deps - with mocksession.newaction(venv.name, "update") as action: - venv.update(action) - venv.installpkg(pkg, action) - - -def test_conda_spec(tmpdir, newconfig, mocksession): - """Test environment creation when conda_spec given""" - txt = tmpdir.join("conda-spec.txt") - txt.write( - """ - pytest - """ - ) - config = newconfig( - [], - """ - [testenv:py123] - conda_deps= - numpy - astropy - conda_spec={} - """.format( - str(txt) - ), - ) - venv, action, pcalls = create_test_env(config, mocksession, "py123") - - assert venv.envconfig.conda_spec - assert len(venv.envconfig.conda_deps) == 2 - - tox_testenv_install_deps(action=action, venv=venv) - # We expect conda_spec to be appended to conda deps install - assert len(pcalls) >= 1 - call = pcalls[-1] - conda_cmd = call.args - assert "conda" in os.path.split(conda_cmd[0])[-1] - assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] - # Make sure that python is explicitly given as part of every conda install - # in order to avoid inadvertent upgrades of python itself. - assert conda_cmd[6].startswith("python=") - assert conda_cmd[7:9] == ["numpy", "astropy"] - assert conda_cmd[-1].startswith("--file") - assert conda_cmd[-1].endswith("conda-spec.txt") - - -def test_empty_conda_spec_and_env(tmpdir, newconfig, mocksession): - """Test environment creation when empty conda_spec and conda_env.""" - txt = tmpdir.join("conda-spec.txt") - txt.write( - """ - pytest - """ - ) - config = newconfig( - [], - """ - [testenv:py123] - conda_env= - foo: path-to.yml - conda_spec= - foo: path-to.yml - """, - ) - venv, _, _ = create_test_env(config, mocksession, "py123") - - assert venv.envconfig.conda_spec is None - assert venv.envconfig.conda_env is None - - -def test_conda_env(tmpdir, newconfig, mocksession): - """Test environment creation when conda_env given""" - yml = tmpdir.join("conda-env.yml") - yml.write( - """ - name: tox-conda - channels: - - conda-forge - - nodefaults - dependencies: - - numpy - - astropy - - pip: - - pytest - """ - ) - config = newconfig( - [], - """ - [testenv:py123] - conda_env={} - """.format( - str(yml) - ), - ) - - venv = VirtualEnv(config.envconfigs["py123"]) - assert venv.path == config.envconfigs["py123"].envdir - - venv, action, pcalls = create_test_env(config, mocksession, "py123") - assert venv.envconfig.conda_env - - mock_file = mock_open() - with patch("tox_conda.plugin.tempfile.NamedTemporaryFile", mock_file): - with patch.object(pathlib.Path, "unlink", autospec=True) as mock_unlink: - with mocksession.newaction(venv.name, "getenv") as action: - tox_testenv_create(action=action, venv=venv) - mock_unlink.assert_called_once - - mock_file.assert_called_with(dir=tmpdir, prefix="tox_conda_tmp", suffix=".yaml", delete=False) - - pcalls = mocksession._pcalls - assert len(pcalls) >= 1 - call = pcalls[-1] - cmd = call.args - assert "conda" in os.path.split(cmd[0])[-1] - assert cmd[1:4] == ["env", "create", "-p"] - assert venv.path == call.args[4] - assert call.args[5].startswith("--file") - assert cmd[6] == str(mock_file().name) - - yaml = YAML() - tmp_env = yaml.load(mock_open_to_string(mock_file)) - assert tmp_env["dependencies"][-1].startswith("python=") - - -def test_conda_env_and_spec(tmpdir, newconfig, mocksession): - """Test environment creation when conda_env and conda_spec are given""" - yml = tmpdir.join("conda-env.yml") - yml.write( - """ - name: tox-conda - channels: - - conda-forge - - nodefaults - dependencies: - - numpy - - astropy - """ - ) - txt = tmpdir.join("conda-spec.txt") - txt.write( - """ - pytest - """ - ) - config = newconfig( - [], - """ - [testenv:py123] - conda_env={} - conda_spec={} - """.format( - str(yml), str(txt) - ), - ) - venv, action, pcalls = create_test_env(config, mocksession, "py123") - - assert venv.envconfig.conda_env - assert venv.envconfig.conda_spec - - mock_file = mock_open() - with patch("tox_conda.plugin.tempfile.NamedTemporaryFile", mock_file): - with patch.object(pathlib.Path, "unlink", autospec=True) as mock_unlink: - with mocksession.newaction(venv.name, "getenv") as action: - tox_testenv_create(action=action, venv=venv) - mock_unlink.assert_called_once - - mock_file.assert_called_with(dir=tmpdir, prefix="tox_conda_tmp", suffix=".yaml", delete=False) - - pcalls = mocksession._pcalls - assert len(pcalls) >= 1 - call = pcalls[-1] - cmd = call.args - assert "conda" in os.path.split(cmd[0])[-1] - assert cmd[1:4] == ["env", "create", "-p"] - assert venv.path == call.args[4] - assert call.args[5].startswith("--file") - assert cmd[6] == str(mock_file().name) - - yaml = YAML() - tmp_env = yaml.load(mock_open_to_string(mock_file)) - assert tmp_env["dependencies"][-1].startswith("python=") - - with mocksession.newaction(venv.name, "getenv") as action: - tox_testenv_install_deps(action=action, venv=venv) - pcalls = mocksession._pcalls - # We expect conda_spec to be appended to conda deps install - assert len(pcalls) >= 1 - call = pcalls[-1] - conda_cmd = call.args - assert "conda" in os.path.split(conda_cmd[0])[-1] - assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] - # Make sure that python is explicitly given as part of every conda install - # in order to avoid inadvertent upgrades of python itself. - assert conda_cmd[6].startswith("python=") - assert conda_cmd[-1].startswith("--file") - assert conda_cmd[-1].endswith("conda-spec.txt") - - -def test_conda_install_args(newconfig, mocksession): - config = newconfig( - [], - """ - [testenv:py123] - conda_deps= - numpy - conda_install_args= - --override-channels - """, - ) - - venv, action, pcalls = create_test_env(config, mocksession, "py123") - - assert len(venv.envconfig.conda_install_args) == 1 - - tox_testenv_install_deps(action=action, venv=venv) - - call = pcalls[-1] - assert call.args[6] == "--override-channels" - - -def test_conda_create_args(newconfig, mocksession): - config = newconfig( - [], - """ - [testenv:py123] - conda_create_args= - --override-channels - """, - ) - - venv = VirtualEnv(config.envconfigs["py123"]) - assert venv.path == config.envconfigs["py123"].envdir - - with mocksession.newaction(venv.name, "getenv") as action: - tox_testenv_create(action=action, venv=venv) - pcalls = mocksession._pcalls - assert len(pcalls) >= 1 - call = pcalls[-1] - assert "conda" in call.args[0] - assert "create" == call.args[1] - assert "--yes" == call.args[2] - assert "-p" == call.args[3] - assert venv.path == call.args[4] - assert call.args[5] == "--override-channels" - assert call.args[6].startswith("python=") - - -def test_verbosity(newconfig, mocksession): - config = newconfig( - [], - """ - [testenv:py1] - conda_deps=numpy - [testenv:py2] - conda_deps=numpy - """, - ) - - venv, action, pcalls = create_test_env(config, mocksession, "py1") - tox_testenv_install_deps(action=action, venv=venv) - assert len(pcalls) == 1 - call = pcalls[0] - assert "conda" in call.args[0] - assert "install" == call.args[1] - assert isinstance(call.stdout, io.IOBase) - - tox.reporter.update_default_reporter( - tox.reporter.Verbosity.DEFAULT, tox.reporter.Verbosity.DEBUG - ) - venv, action, pcalls = create_test_env(config, mocksession, "py2") - tox_testenv_install_deps(action=action, venv=venv) - assert len(pcalls) == 1 - call = pcalls[0] - assert "conda" in call.args[0] - assert "install" == call.args[1] - assert not isinstance(call.stdout, io.IOBase) - - -def mock_open_to_string(mock): - return "".join(call.args[0] for call in mock().write.call_args_list) +from tox.pytest import CaptureFixture, ToxProject, ToxProjectCreator +from tox_conda.plugin import CondaEnvRunner + +import tox.run +from tox.config.sets import EnvConfigSet +from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteStatus, Outcome +from tox.execute.request import ExecuteRequest, shell_cmd +from tox.execute.stream import SyncWrite +from tox.plugin import manager +from tox.report import LOGGER, OutErr +from tox.run import run as tox_run +from tox.run import setup_state as previous_setup_state +from tox.session.cmd.run.parallel import ENV_VAR_KEY +from tox.session.state import State +from tox.tox_env import api as tox_env_api +from tox.tox_env.api import ToxEnv + +# from tox_conda.plugin import tox_testenv_create, tox_testenv_install_deps + +# pytest_plugins = "tox.pytest" + + +@pytest.fixture(name="tox_project") +def init_fixture( + tmp_path: Path, + capfd: CaptureFixture, + monkeypatch: MonkeyPatch, + mocker: MockerFixture, +) -> ToxProjectCreator: + def _init( + files: Dict[str, Any], base: Optional[Path] = None, prj_path: Optional[Path] = None + ) -> ToxProject: + """create tox projects""" + return ToxProject(files, base, prj_path or tmp_path / "p", capfd, monkeypatch, mocker) + + return _init + +# class MockExecute(Execute): +# def __init__(self, colored: bool, exit_code: int) -> None: +# self.exit_code = exit_code +# super().__init__(colored) + +# def build_instance( +# self, +# request: ExecuteRequest, +# options: ExecuteOptions, +# out: SyncWrite, +# err: SyncWrite, +# ) -> ExecuteInstance: +# return MockExecuteInstance(request, options, out, err, self.exit_code) + + +@pytest.fixture +def mock_conda_env_runner(request, monkeypatch): + class MockExecuteStatus(ExecuteStatus): + def __init__(self, options: ExecuteOptions, out: SyncWrite, err: SyncWrite, exit_code: int) -> None: + super().__init__(options, out, err) + self._exit_code = exit_code + + @property + def exit_code(self) -> Optional[int]: + return self._exit_code + + def wait(self, timeout: Optional[float] = None) -> Optional[int]: # noqa: U100 + return self._exit_code + + def write_stdin(self, content: str) -> None: # noqa: U100 + return None # pragma: no cover + + def interrupt(self) -> None: + return None # pragma: no cover + + class MockExecuteInstance(ExecuteInstance): + def __init__( + self, + request: ExecuteRequest, + options: ExecuteOptions, + out: SyncWrite, + err: SyncWrite, + exit_code: int, + ) -> None: + super().__init__(request, options, out, err) + self.exit_code = exit_code + + def __enter__(self) -> ExecuteStatus: + return MockExecuteStatus(self.options, self._out, self._err, self.exit_code) + + def __exit__( + self, + exc_type: Optional[BaseException], # noqa: U100 + exc_val: Optional[BaseException], # noqa: U100 + exc_tb: Optional[TracebackType], # noqa: U100 + ) -> None: + pass + + @property + def cmd(self) -> Sequence[str]: + return self.request.cmd + + + shell_cmds = [] + mocked_run_ids = request.param + original_execute_instance_factor = CondaEnvRunner._execute_instance_factory + + def mock_execute_instance_factory(request: ExecuteRequest, + options: ExecuteOptions, + out: SyncWrite, + err: SyncWrite): + shell_cmds.append(request.cmd) + + if request.run_id in mocked_run_ids: + return MockExecuteInstance(request, options, out, err, 0) + else: + return original_execute_instance_factor(request, options, out, err) + + # CondaEnvRunner._execute_instance_factory = mock_execute_instance_factory + + monkeypatch.setattr(CondaEnvRunner, "_execute_instance_factory", mock_execute_instance_factory) + + yield shell_cmds + +@pytest.mark.parametrize('mock_conda_env_runner', [["create_python_env-create", "create_python_env-install", "install_package"]], indirect=True) +def test_conda_create(tox_project, monkeypatch, mock_conda_env_runner): + ini = "[testenv:py123]" + outcome = tox_project({"tox.ini": ini}).run("-e", "py123") + executed_shell_commands = mock_conda_env_runner + outcome.assert_success() + + # ini = "[testenv:py123]" + # tox_proj = tox_project({"tox.ini": ini}) + # execute_calls = tox_proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + # result = tox_proj.run("-e", "py123") + # result.assert_success() + # assert len(execute_calls) == 2 + +# def test_conda_create(tox_project, monkeypatch): +# original_run = CondaEnvRunner._run_with_executor + +# def mock_run(self, executor, request): +# if request.run_id == "_get_python": +# return original_run(self, executor, request) +# # Define your own logic here +# return "mocked return value" + +# monkeypatch.setattr(CondaEnvRunner, "_run_with_executor", mock_run) + +# ini = "[testenv:py123]" +# outcome = tox_project({"tox.ini": ini}).run("-e", "py123") +# outcome.assert_success() + + + +# @pytest.fixture +# def mock_conda_env_runner(request, monkeypatch): +# original_run = CondaEnvRunner._run_with_executor +# shell_cmds = [] + +# def mock_run(self, executor, request): +# shell_cmds.append(request.shell_cmd) +# if request.run_id == "_get_python": +# return original_run(self, executor, request) + +# mocked_values = request.config.getoption("mocked_values", default={}) +# return mocked_values.get(request.run_id, "default mocked return value") + +# def patch_conda_env_runner(): +# monkeypatch.setattr(CondaEnvRunner, "_run_with_executor", mock_run) + +# patch_conda_env_runner() +# yield shell_cmds + +# @pytest.fixture +# def tox_ini(): +# return "[testenv:py123]" + +# def test_conda_create(tox_project, mock_conda_env_runner, tox_ini, pytestconfig): +# ini = tox_ini +# pytestconfig.option.mocked_values = {"py123": "custom mocked value for py123"} + +# outcome = tox_project({"tox.ini": ini}).run("-e", "py123") +# outcome.assert_success() + +# assert "conda env create" in mock_conda_env_runner +# assert outcome.ret == "custom mocked value for py123" + + +# def test_conda_create(tox_project, monkeypatch): +# ini = "[testenv:py123]" +# tox_proj = tox_project({"tox.ini": ini}) +# execute_calls = tox_proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) +# result = tox_proj.run("-e", "py123") +# result.assert_success() +# assert len(execute_calls) == 2 + + # venv = VirtualEnv(config.envconfigs["py123"]) + # assert venv.path == config.envconfigs["py123"].envdir + + # with mocksession.newaction(venv.name, "getenv") as action: + # tox_testenv_create(action=action, venv=venv) + # pcalls = mocksession._pcalls + # assert len(pcalls) >= 1 + # call = pcalls[-1] + # assert "conda" in call.args[0] + # assert "create" == call.args[1] + # assert "--yes" == call.args[2] + # assert "-p" == call.args[3] + # assert venv.path == call.args[4] + # assert call.args[5].startswith("python=") + + +# def create_test_env(config, mocksession, envname): + +# venv = VirtualEnv(config.envconfigs[envname]) +# with mocksession.newaction(venv.name, "getenv") as action: +# tox_testenv_create(action=action, venv=venv) +# pcalls = mocksession._pcalls +# assert len(pcalls) >= 1 +# pcalls[:] = [] + +# return venv, action, pcalls + + +# def test_install_deps_no_conda(newconfig, mocksession, monkeypatch): +# """Test installation using conda when no conda_deps are given""" +# # No longer remove the temporary script, so we can check its contents. +# monkeypatch.delattr(PopenInActivatedEnv, "__del__", raising=False) + +# env_name = "py123" +# config = newconfig( +# [], +# """ +# [testenv:{}] +# deps= +# numpy +# -r requirements.txt +# astropy +# """.format( +# env_name +# ), +# ) + +# config.toxinidir.join("requirements.txt").write("") + +# venv, action, pcalls = create_test_env(config, mocksession, env_name) + +# assert len(venv.envconfig.deps) == 3 +# assert len(venv.envconfig.conda_deps) == 0 + +# tox_testenv_install_deps(action=action, venv=venv) + +# assert len(pcalls) >= 1 + +# call = pcalls[-1] + +# if tox.INFO.IS_WIN: +# script_lines = " ".join(call.args).split(" && ", maxsplit=1) +# pattern = r"conda\.bat activate .*{}".format(re.escape(env_name)) +# else: +# # Get the cmd args from the script. +# shell, cmd_script = call.args +# assert shell == "/bin/sh" +# with open(cmd_script) as stream: +# script_lines = stream.readlines() +# pattern = r"eval \"\$\(/.*/conda shell\.posix activate /.*/{}\)\"".format(env_name) + +# assert re.match(pattern, script_lines[0]) + +# cmd = script_lines[1].split() +# assert cmd[-6:] == ["-m", "pip", "install", "numpy", "-rrequirements.txt", "astropy"] + + +# def test_install_conda_deps(newconfig, mocksession): +# config = newconfig( +# [], +# """ +# [testenv:py123] +# deps= +# numpy +# astropy +# conda_deps= +# pytest +# asdf +# """, +# ) + +# venv, action, pcalls = create_test_env(config, mocksession, "py123") + +# assert len(venv.envconfig.conda_deps) == 2 +# assert len(venv.envconfig.deps) == 2 + len(venv.envconfig.conda_deps) + +# tox_testenv_install_deps(action=action, venv=venv) +# # We expect two calls: one for conda deps, and one for pip deps +# assert len(pcalls) >= 2 +# call = pcalls[-2] +# conda_cmd = call.args +# assert "conda" in os.path.split(conda_cmd[0])[-1] +# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] +# # Make sure that python is explicitly given as part of every conda install +# # in order to avoid inadvertent upgrades of python itself. +# assert conda_cmd[6].startswith("python=") +# assert conda_cmd[7:9] == ["pytest", "asdf"] + + +# def test_install_conda_no_pip(newconfig, mocksession): +# config = newconfig( +# [], +# """ +# [testenv:py123] +# conda_deps= +# pytest +# asdf +# """, +# ) + +# venv, action, pcalls = create_test_env(config, mocksession, "py123") + +# assert len(venv.envconfig.conda_deps) == 2 +# assert len(venv.envconfig.deps) == len(venv.envconfig.conda_deps) + +# tox_testenv_install_deps(action=action, venv=venv) +# # We expect only one call since there are no true pip dependencies +# assert len(pcalls) >= 1 + +# # Just a quick sanity check for the conda install command +# call = pcalls[-1] +# conda_cmd = call.args +# assert "conda" in os.path.split(conda_cmd[0])[-1] +# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] + + +# def test_update(tmpdir, newconfig, mocksession): +# pkg = tmpdir.ensure("package.tar.gz") +# config = newconfig( +# [], +# """ +# [testenv:py123] +# deps= +# numpy +# astropy +# conda_deps= +# pytest +# asdf +# """, +# ) + +# venv, action, pcalls = create_test_env(config, mocksession, "py123") +# tox_testenv_install_deps(action=action, venv=venv) + +# venv.hook.tox_testenv_create = tox_testenv_create +# venv.hook.tox_testenv_install_deps = tox_testenv_install_deps +# with mocksession.newaction(venv.name, "update") as action: +# venv.update(action) +# venv.installpkg(pkg, action) + + +# def test_conda_spec(tmpdir, newconfig, mocksession): +# """Test environment creation when conda_spec given""" +# txt = tmpdir.join("conda-spec.txt") +# txt.write( +# """ +# pytest +# """ +# ) +# config = newconfig( +# [], +# """ +# [testenv:py123] +# conda_deps= +# numpy +# astropy +# conda_spec={} +# """.format( +# str(txt) +# ), +# ) +# venv, action, pcalls = create_test_env(config, mocksession, "py123") + +# assert venv.envconfig.conda_spec +# assert len(venv.envconfig.conda_deps) == 2 + +# tox_testenv_install_deps(action=action, venv=venv) +# # We expect conda_spec to be appended to conda deps install +# assert len(pcalls) >= 1 +# call = pcalls[-1] +# conda_cmd = call.args +# assert "conda" in os.path.split(conda_cmd[0])[-1] +# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] +# # Make sure that python is explicitly given as part of every conda install +# # in order to avoid inadvertent upgrades of python itself. +# assert conda_cmd[6].startswith("python=") +# assert conda_cmd[7:9] == ["numpy", "astropy"] +# assert conda_cmd[-1].startswith("--file") +# assert conda_cmd[-1].endswith("conda-spec.txt") + + +# def test_empty_conda_spec_and_env(tmpdir, newconfig, mocksession): +# """Test environment creation when empty conda_spec and conda_env.""" +# txt = tmpdir.join("conda-spec.txt") +# txt.write( +# """ +# pytest +# """ +# ) +# config = newconfig( +# [], +# """ +# [testenv:py123] +# conda_env= +# foo: path-to.yml +# conda_spec= +# foo: path-to.yml +# """, +# ) +# venv, _, _ = create_test_env(config, mocksession, "py123") + +# assert venv.envconfig.conda_spec is None +# assert venv.envconfig.conda_env is None + + +# def test_conda_env(tmpdir, newconfig, mocksession): +# """Test environment creation when conda_env given""" +# yml = tmpdir.join("conda-env.yml") +# yml.write( +# """ +# name: tox-conda +# channels: +# - conda-forge +# - nodefaults +# dependencies: +# - numpy +# - astropy +# - pip: +# - pytest +# """ +# ) +# config = newconfig( +# [], +# """ +# [testenv:py123] +# conda_env={} +# """.format( +# str(yml) +# ), +# ) + +# venv = VirtualEnv(config.envconfigs["py123"]) +# assert venv.path == config.envconfigs["py123"].envdir + +# venv, action, pcalls = create_test_env(config, mocksession, "py123") +# assert venv.envconfig.conda_env + +# mock_file = mock_open() +# with patch("tox_conda.plugin.tempfile.NamedTemporaryFile", mock_file): +# with patch.object(pathlib.Path, "unlink", autospec=True) as mock_unlink: +# with mocksession.newaction(venv.name, "getenv") as action: +# tox_testenv_create(action=action, venv=venv) +# mock_unlink.assert_called_once + +# mock_file.assert_called_with(dir=tmpdir, prefix="tox_conda_tmp", suffix=".yaml", delete=False) + +# pcalls = mocksession._pcalls +# assert len(pcalls) >= 1 +# call = pcalls[-1] +# cmd = call.args +# assert "conda" in os.path.split(cmd[0])[-1] +# assert cmd[1:4] == ["env", "create", "-p"] +# assert venv.path == call.args[4] +# assert call.args[5].startswith("--file") +# assert cmd[6] == str(mock_file().name) + +# yaml = YAML() +# tmp_env = yaml.load(mock_open_to_string(mock_file)) +# assert tmp_env["dependencies"][-1].startswith("python=") + + +# def test_conda_env_and_spec(tmpdir, newconfig, mocksession): +# """Test environment creation when conda_env and conda_spec are given""" +# yml = tmpdir.join("conda-env.yml") +# yml.write( +# """ +# name: tox-conda +# channels: +# - conda-forge +# - nodefaults +# dependencies: +# - numpy +# - astropy +# """ +# ) +# txt = tmpdir.join("conda-spec.txt") +# txt.write( +# """ +# pytest +# """ +# ) +# config = newconfig( +# [], +# """ +# [testenv:py123] +# conda_env={} +# conda_spec={} +# """.format( +# str(yml), str(txt) +# ), +# ) +# venv, action, pcalls = create_test_env(config, mocksession, "py123") + +# assert venv.envconfig.conda_env +# assert venv.envconfig.conda_spec + +# mock_file = mock_open() +# with patch("tox_conda.plugin.tempfile.NamedTemporaryFile", mock_file): +# with patch.object(pathlib.Path, "unlink", autospec=True) as mock_unlink: +# with mocksession.newaction(venv.name, "getenv") as action: +# tox_testenv_create(action=action, venv=venv) +# mock_unlink.assert_called_once + +# mock_file.assert_called_with(dir=tmpdir, prefix="tox_conda_tmp", suffix=".yaml", delete=False) + +# pcalls = mocksession._pcalls +# assert len(pcalls) >= 1 +# call = pcalls[-1] +# cmd = call.args +# assert "conda" in os.path.split(cmd[0])[-1] +# assert cmd[1:4] == ["env", "create", "-p"] +# assert venv.path == call.args[4] +# assert call.args[5].startswith("--file") +# assert cmd[6] == str(mock_file().name) + +# yaml = YAML() +# tmp_env = yaml.load(mock_open_to_string(mock_file)) +# assert tmp_env["dependencies"][-1].startswith("python=") + +# with mocksession.newaction(venv.name, "getenv") as action: +# tox_testenv_install_deps(action=action, venv=venv) +# pcalls = mocksession._pcalls +# # We expect conda_spec to be appended to conda deps install +# assert len(pcalls) >= 1 +# call = pcalls[-1] +# conda_cmd = call.args +# assert "conda" in os.path.split(conda_cmd[0])[-1] +# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] +# # Make sure that python is explicitly given as part of every conda install +# # in order to avoid inadvertent upgrades of python itself. +# assert conda_cmd[6].startswith("python=") +# assert conda_cmd[-1].startswith("--file") +# assert conda_cmd[-1].endswith("conda-spec.txt") + + +# def test_conda_install_args(newconfig, mocksession): +# config = newconfig( +# [], +# """ +# [testenv:py123] +# conda_deps= +# numpy +# conda_install_args= +# --override-channels +# """, +# ) + +# venv, action, pcalls = create_test_env(config, mocksession, "py123") + +# assert len(venv.envconfig.conda_install_args) == 1 + +# tox_testenv_install_deps(action=action, venv=venv) + +# call = pcalls[-1] +# assert call.args[6] == "--override-channels" + + +# def test_conda_create_args(newconfig, mocksession): +# config = newconfig( +# [], +# """ +# [testenv:py123] +# conda_create_args= +# --override-channels +# """, +# ) + +# venv = VirtualEnv(config.envconfigs["py123"]) +# assert venv.path == config.envconfigs["py123"].envdir + +# with mocksession.newaction(venv.name, "getenv") as action: +# tox_testenv_create(action=action, venv=venv) +# pcalls = mocksession._pcalls +# assert len(pcalls) >= 1 +# call = pcalls[-1] +# assert "conda" in call.args[0] +# assert "create" == call.args[1] +# assert "--yes" == call.args[2] +# assert "-p" == call.args[3] +# assert venv.path == call.args[4] +# assert call.args[5] == "--override-channels" +# assert call.args[6].startswith("python=") + + +# def test_verbosity(newconfig, mocksession): +# config = newconfig( +# [], +# """ +# [testenv:py1] +# conda_deps=numpy +# [testenv:py2] +# conda_deps=numpy +# """, +# ) + +# venv, action, pcalls = create_test_env(config, mocksession, "py1") +# tox_testenv_install_deps(action=action, venv=venv) +# assert len(pcalls) == 1 +# call = pcalls[0] +# assert "conda" in call.args[0] +# assert "install" == call.args[1] +# assert isinstance(call.stdout, io.IOBase) + +# tox.reporter.update_default_reporter( +# tox.reporter.Verbosity.DEFAULT, tox.reporter.Verbosity.DEBUG +# ) +# venv, action, pcalls = create_test_env(config, mocksession, "py2") +# tox_testenv_install_deps(action=action, venv=venv) +# assert len(pcalls) == 1 +# call = pcalls[0] +# assert "conda" in call.args[0] +# assert "install" == call.args[1] +# assert not isinstance(call.stdout, io.IOBase) + + +# def mock_open_to_string(mock): +# return "".join(call.args[0] for call in mock().write.call_args_list) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 219a8e0..6e860ea 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -11,7 +11,7 @@ from io import BytesIO, TextIOWrapper from pathlib import Path from time import sleep -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Union from ruamel.yaml import YAML from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteRequest, SyncWrite @@ -30,6 +30,16 @@ class CondaEnvRunner(PythonRun): + # Can be overridden in tests + @staticmethod + def _execute_instance_factory(request: ExecuteRequest, + options: ExecuteOptions, + out: SyncWrite, + err: SyncWrite): + return LocalSubProcessExecuteInstance(request, options, out, err) + + _execute_instance_factory: Union[ExecuteInstance, Callable] = _execute_instance_factory + def __init__(self, create_args: ToxEnvCreateArgs) -> None: self._installer = None self._executor = None @@ -258,8 +268,18 @@ def _generate_install_command(conda_exe: Path, python: str, conda_cache_conf: Di @property def external_executor(self) -> Execute: + class CondaExecutor(LocalSubProcessExecutor): + def build_instance( + self, + request: ExecuteRequest, + options: ExecuteOptions, + out: SyncWrite, + err: SyncWrite, + ) -> ExecuteInstance: + return CondaEnvRunner._execute_instance_factory(request, options, out, err) + if self._external_executor is None: - self._external_executor = LocalSubProcessExecutor(self.options.is_colored) + self._external_executor = CondaExecutor(self.options.is_colored) return self._external_executor @property @@ -291,7 +311,9 @@ def build_instance( request.run_id, request.allow, ) - return LocalSubProcessExecuteInstance(conda_request, options, out, err) + # This creates a LocalSubProcessExecuteInstance in real environment, + # and it allows testing with dependency injection. + return CondaEnvRunner._execute_instance_factory(conda_request, options, out, err) if self._executor is None: self._executor = CondaExecutor(self.options.is_colored) @@ -345,7 +367,7 @@ def _run_in_conda(self, cmd: List[str], run_id: str): run_id, ) - return self._run_with_executor(self.executor, request) + return self._call_executor(self.executor, request) def _run_pure(self, cmd: List[str], run_id: str): request = ExecuteRequest( @@ -356,9 +378,9 @@ def _run_pure(self, cmd: List[str], run_id: str): run_id, ) - return self._run_with_executor(self.external_executor, request) + return self._call_executor(self.external_executor, request) - def _run_with_executor(self, executor: Execute, request: ExecuteRequest): + def _call_executor(self, executor: Execute, request: ExecuteRequest): class NamedBytesIO(BytesIO): def __init__(self, name): self.name = name @@ -401,8 +423,8 @@ def _ensure_python_env_exists(self) -> None: self._created = True else: raise Fail( - f"{self.env_dir} already exists, but it is not a conda environment. Delete in" - " manually first." + f"{self.env_dir} already exists, but it is not a conda environment. " + "Delete it first manually." ) From 503e1902c56ea4718e6accc0b3c8f154e95132d6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:01:08 +0000 Subject: [PATCH 17/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_env.py | 66 ++++++++++++++++++++++------------------- tox_conda/plugin.py | 7 ++--- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 4204a61..addaafe 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -4,26 +4,22 @@ import re import subprocess from pathlib import Path -from typing import Any, Callable, Dict, Optional, Union, Sequence -from unittest.mock import mock_open, patch - - from types import ModuleType, TracebackType +from typing import Any, Callable, Dict, Optional, Sequence, Union +from unittest.mock import mock_open, patch import pytest import tox +import tox.run from pytest import MonkeyPatch from pytest_mock import MockerFixture from ruamel.yaml import YAML -from tox.pytest import CaptureFixture, ToxProject, ToxProjectCreator -from tox_conda.plugin import CondaEnvRunner - -import tox.run from tox.config.sets import EnvConfigSet from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteStatus, Outcome from tox.execute.request import ExecuteRequest, shell_cmd from tox.execute.stream import SyncWrite from tox.plugin import manager +from tox.pytest import CaptureFixture, ToxProject, ToxProjectCreator from tox.report import LOGGER, OutErr from tox.run import run as tox_run from tox.run import setup_state as previous_setup_state @@ -32,6 +28,8 @@ from tox.tox_env import api as tox_env_api from tox.tox_env.api import ToxEnv +from tox_conda.plugin import CondaEnvRunner + # from tox_conda.plugin import tox_testenv_create, tox_testenv_install_deps # pytest_plugins = "tox.pytest" @@ -52,6 +50,7 @@ def _init( return _init + # class MockExecute(Execute): # def __init__(self, colored: bool, exit_code: int) -> None: # self.exit_code = exit_code @@ -70,7 +69,9 @@ def _init( @pytest.fixture def mock_conda_env_runner(request, monkeypatch): class MockExecuteStatus(ExecuteStatus): - def __init__(self, options: ExecuteOptions, out: SyncWrite, err: SyncWrite, exit_code: int) -> None: + def __init__( + self, options: ExecuteOptions, out: SyncWrite, err: SyncWrite, exit_code: int + ) -> None: super().__init__(options, out, err) self._exit_code = exit_code @@ -113,30 +114,33 @@ def __exit__( @property def cmd(self) -> Sequence[str]: return self.request.cmd - shell_cmds = [] mocked_run_ids = request.param original_execute_instance_factor = CondaEnvRunner._execute_instance_factory - def mock_execute_instance_factory(request: ExecuteRequest, - options: ExecuteOptions, - out: SyncWrite, - err: SyncWrite): + def mock_execute_instance_factory( + request: ExecuteRequest, options: ExecuteOptions, out: SyncWrite, err: SyncWrite + ): shell_cmds.append(request.cmd) if request.run_id in mocked_run_ids: return MockExecuteInstance(request, options, out, err, 0) else: return original_execute_instance_factor(request, options, out, err) - + # CondaEnvRunner._execute_instance_factory = mock_execute_instance_factory monkeypatch.setattr(CondaEnvRunner, "_execute_instance_factory", mock_execute_instance_factory) yield shell_cmds -@pytest.mark.parametrize('mock_conda_env_runner', [["create_python_env-create", "create_python_env-install", "install_package"]], indirect=True) + +@pytest.mark.parametrize( + "mock_conda_env_runner", + [["create_python_env-create", "create_python_env-install", "install_package"]], + indirect=True, +) def test_conda_create(tox_project, monkeypatch, mock_conda_env_runner): ini = "[testenv:py123]" outcome = tox_project({"tox.ini": ini}).run("-e", "py123") @@ -150,6 +154,7 @@ def test_conda_create(tox_project, monkeypatch, mock_conda_env_runner): # result.assert_success() # assert len(execute_calls) == 2 + # def test_conda_create(tox_project, monkeypatch): # original_run = CondaEnvRunner._run_with_executor @@ -166,7 +171,6 @@ def test_conda_create(tox_project, monkeypatch, mock_conda_env_runner): # outcome.assert_success() - # @pytest.fixture # def mock_conda_env_runner(request, monkeypatch): # original_run = CondaEnvRunner._run_with_executor @@ -209,20 +213,20 @@ def test_conda_create(tox_project, monkeypatch, mock_conda_env_runner): # result.assert_success() # assert len(execute_calls) == 2 - # venv = VirtualEnv(config.envconfigs["py123"]) - # assert venv.path == config.envconfigs["py123"].envdir - - # with mocksession.newaction(venv.name, "getenv") as action: - # tox_testenv_create(action=action, venv=venv) - # pcalls = mocksession._pcalls - # assert len(pcalls) >= 1 - # call = pcalls[-1] - # assert "conda" in call.args[0] - # assert "create" == call.args[1] - # assert "--yes" == call.args[2] - # assert "-p" == call.args[3] - # assert venv.path == call.args[4] - # assert call.args[5].startswith("python=") +# venv = VirtualEnv(config.envconfigs["py123"]) +# assert venv.path == config.envconfigs["py123"].envdir + +# with mocksession.newaction(venv.name, "getenv") as action: +# tox_testenv_create(action=action, venv=venv) +# pcalls = mocksession._pcalls +# assert len(pcalls) >= 1 +# call = pcalls[-1] +# assert "conda" in call.args[0] +# assert "create" == call.args[1] +# assert "--yes" == call.args[2] +# assert "-p" == call.args[3] +# assert venv.path == call.args[4] +# assert call.args[5].startswith("python=") # def create_test_env(config, mocksession, envname): diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 6e860ea..76f250c 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -32,10 +32,9 @@ class CondaEnvRunner(PythonRun): # Can be overridden in tests @staticmethod - def _execute_instance_factory(request: ExecuteRequest, - options: ExecuteOptions, - out: SyncWrite, - err: SyncWrite): + def _execute_instance_factory( + request: ExecuteRequest, options: ExecuteOptions, out: SyncWrite, err: SyncWrite + ): return LocalSubProcessExecuteInstance(request, options, out, err) _execute_instance_factory: Union[ExecuteInstance, Callable] = _execute_instance_factory From b2c68cdabfc335935b4471b1dfbe950ed4196a5b Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Thu, 1 Jun 2023 21:13:24 +0100 Subject: [PATCH 18/39] Fix test passing. --- tests/conftest.py | 122 ++++++++++++++++++++++- tests/test_conda_env.py | 208 +++------------------------------------- 2 files changed, 136 insertions(+), 194 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ee2f10c..5e6f628 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,121 @@ -# from tox._pytestplugin import * # noqa +import io +import os +import pathlib +import re +import subprocess +from pathlib import Path +from types import ModuleType, TracebackType +from typing import Any, Callable, Dict, Optional, Sequence, Union +from unittest.mock import mock_open, patch + +import pytest +import tox +import tox.run +from pytest import MonkeyPatch +from pytest_mock import MockerFixture +from ruamel.yaml import YAML +from tox.config.sets import EnvConfigSet +from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteStatus, Outcome +from tox.execute.request import ExecuteRequest, shell_cmd +from tox.execute.stream import SyncWrite +from tox.plugin import manager +from tox.pytest import CaptureFixture, ToxProject, ToxProjectCreator +from tox.report import LOGGER, OutErr +from tox.run import run as tox_run +from tox.run import setup_state as previous_setup_state +from tox.session.cmd.run.parallel import ENV_VAR_KEY +from tox.session.state import State +from tox.tox_env import api as tox_env_api +from tox.tox_env.api import ToxEnv + +from tox_conda.plugin import CondaEnvRunner + +# from tox_conda.plugin import tox_testenv_create, tox_testenv_install_deps + +# pytest_plugins = "tox.pytest" + + +@pytest.fixture(name="tox_project") +def init_fixture( + tmp_path: Path, + capfd: CaptureFixture, + monkeypatch: MonkeyPatch, + mocker: MockerFixture, +) -> ToxProjectCreator: + def _init( + files: Dict[str, Any], base: Optional[Path] = None, prj_path: Optional[Path] = None + ) -> ToxProject: + """create tox projects""" + return ToxProject(files, base, prj_path or tmp_path / "p", capfd, monkeypatch, mocker) + + return _init + +@pytest.fixture +def mock_conda_env_runner(request, monkeypatch): + class MockExecuteStatus(ExecuteStatus): + def __init__( + self, options: ExecuteOptions, out: SyncWrite, err: SyncWrite, exit_code: int + ) -> None: + super().__init__(options, out, err) + self._exit_code = exit_code + + @property + def exit_code(self) -> Optional[int]: + return self._exit_code + + def wait(self, timeout: Optional[float] = None) -> Optional[int]: # noqa: U100 + return self._exit_code + + def write_stdin(self, content: str) -> None: # noqa: U100 + return None # pragma: no cover + + def interrupt(self) -> None: + return None # pragma: no cover + + class MockExecuteInstance(ExecuteInstance): + def __init__( + self, + request: ExecuteRequest, + options: ExecuteOptions, + out: SyncWrite, + err: SyncWrite, + exit_code: int, + ) -> None: + super().__init__(request, options, out, err) + self.exit_code = exit_code + + def __enter__(self) -> ExecuteStatus: + return MockExecuteStatus(self.options, self._out, self._err, self.exit_code) + + def __exit__( + self, + exc_type: Optional[BaseException], # noqa: U100 + exc_val: Optional[BaseException], # noqa: U100 + exc_tb: Optional[TracebackType], # noqa: U100 + ) -> None: + pass + + @property + def cmd(self) -> Sequence[str]: + return self.request.cmd + + shell_cmds = [] + no_mocked_run_ids = getattr(request, 'param', None) + if no_mocked_run_ids is None: + no_mocked_run_ids = ["_get_python"] + original_execute_instance_factor = CondaEnvRunner._execute_instance_factory + + def mock_execute_instance_factory( + request: ExecuteRequest, options: ExecuteOptions, out: SyncWrite, err: SyncWrite + ): + shell_cmds.append(request.cmd) + + if request.run_id not in no_mocked_run_ids: + return MockExecuteInstance(request, options, out, err, 0) + else: + return original_execute_instance_factor(request, options, out, err) + + monkeypatch.setattr(CondaEnvRunner, "_execute_instance_factory", mock_execute_instance_factory) + + yield shell_cmds + diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index addaafe..035e796 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -30,204 +30,26 @@ from tox_conda.plugin import CondaEnvRunner -# from tox_conda.plugin import tox_testenv_create, tox_testenv_install_deps - -# pytest_plugins = "tox.pytest" - - -@pytest.fixture(name="tox_project") -def init_fixture( - tmp_path: Path, - capfd: CaptureFixture, - monkeypatch: MonkeyPatch, - mocker: MockerFixture, -) -> ToxProjectCreator: - def _init( - files: Dict[str, Any], base: Optional[Path] = None, prj_path: Optional[Path] = None - ) -> ToxProject: - """create tox projects""" - return ToxProject(files, base, prj_path or tmp_path / "p", capfd, monkeypatch, mocker) - - return _init - - -# class MockExecute(Execute): -# def __init__(self, colored: bool, exit_code: int) -> None: -# self.exit_code = exit_code -# super().__init__(colored) - -# def build_instance( -# self, -# request: ExecuteRequest, -# options: ExecuteOptions, -# out: SyncWrite, -# err: SyncWrite, -# ) -> ExecuteInstance: -# return MockExecuteInstance(request, options, out, err, self.exit_code) - - -@pytest.fixture -def mock_conda_env_runner(request, monkeypatch): - class MockExecuteStatus(ExecuteStatus): - def __init__( - self, options: ExecuteOptions, out: SyncWrite, err: SyncWrite, exit_code: int - ) -> None: - super().__init__(options, out, err) - self._exit_code = exit_code - - @property - def exit_code(self) -> Optional[int]: - return self._exit_code - - def wait(self, timeout: Optional[float] = None) -> Optional[int]: # noqa: U100 - return self._exit_code - - def write_stdin(self, content: str) -> None: # noqa: U100 - return None # pragma: no cover - - def interrupt(self) -> None: - return None # pragma: no cover - - class MockExecuteInstance(ExecuteInstance): - def __init__( - self, - request: ExecuteRequest, - options: ExecuteOptions, - out: SyncWrite, - err: SyncWrite, - exit_code: int, - ) -> None: - super().__init__(request, options, out, err) - self.exit_code = exit_code - - def __enter__(self) -> ExecuteStatus: - return MockExecuteStatus(self.options, self._out, self._err, self.exit_code) - - def __exit__( - self, - exc_type: Optional[BaseException], # noqa: U100 - exc_val: Optional[BaseException], # noqa: U100 - exc_tb: Optional[TracebackType], # noqa: U100 - ) -> None: - pass - - @property - def cmd(self) -> Sequence[str]: - return self.request.cmd - - shell_cmds = [] - mocked_run_ids = request.param - original_execute_instance_factor = CondaEnvRunner._execute_instance_factory - - def mock_execute_instance_factory( - request: ExecuteRequest, options: ExecuteOptions, out: SyncWrite, err: SyncWrite - ): - shell_cmds.append(request.cmd) - - if request.run_id in mocked_run_ids: - return MockExecuteInstance(request, options, out, err, 0) - else: - return original_execute_instance_factor(request, options, out, err) - - # CondaEnvRunner._execute_instance_factory = mock_execute_instance_factory - - monkeypatch.setattr(CondaEnvRunner, "_execute_instance_factory", mock_execute_instance_factory) - - yield shell_cmds - - -@pytest.mark.parametrize( - "mock_conda_env_runner", - [["create_python_env-create", "create_python_env-install", "install_package"]], - indirect=True, -) -def test_conda_create(tox_project, monkeypatch, mock_conda_env_runner): - ini = "[testenv:py123]" - outcome = tox_project({"tox.ini": ini}).run("-e", "py123") - executed_shell_commands = mock_conda_env_runner - outcome.assert_success() - - # ini = "[testenv:py123]" - # tox_proj = tox_project({"tox.ini": ini}) - # execute_calls = tox_proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) - # result = tox_proj.run("-e", "py123") - # result.assert_success() - # assert len(execute_calls) == 2 - - -# def test_conda_create(tox_project, monkeypatch): -# original_run = CondaEnvRunner._run_with_executor - -# def mock_run(self, executor, request): -# if request.run_id == "_get_python": -# return original_run(self, executor, request) -# # Define your own logic here -# return "mocked return value" - -# monkeypatch.setattr(CondaEnvRunner, "_run_with_executor", mock_run) - -# ini = "[testenv:py123]" -# outcome = tox_project({"tox.ini": ini}).run("-e", "py123") -# outcome.assert_success() - - -# @pytest.fixture -# def mock_conda_env_runner(request, monkeypatch): -# original_run = CondaEnvRunner._run_with_executor -# shell_cmds = [] - -# def mock_run(self, executor, request): -# shell_cmds.append(request.shell_cmd) -# if request.run_id == "_get_python": -# return original_run(self, executor, request) - -# mocked_values = request.config.getoption("mocked_values", default={}) -# return mocked_values.get(request.run_id, "default mocked return value") - -# def patch_conda_env_runner(): -# monkeypatch.setattr(CondaEnvRunner, "_run_with_executor", mock_run) - -# patch_conda_env_runner() -# yield shell_cmds - -# @pytest.fixture -# def tox_ini(): -# return "[testenv:py123]" - -# def test_conda_create(tox_project, mock_conda_env_runner, tox_ini, pytestconfig): -# ini = tox_ini -# pytestconfig.option.mocked_values = {"py123": "custom mocked value for py123"} - -# outcome = tox_project({"tox.ini": ini}).run("-e", "py123") -# outcome.assert_success() - -# assert "conda env create" in mock_conda_env_runner -# assert outcome.ret == "custom mocked value for py123" +def test_conda_create(tox_project, mock_conda_env_runner): + ini = "[testenv:py123]" + proj = tox_project({"tox.ini": ini}) -# def test_conda_create(tox_project, monkeypatch): -# ini = "[testenv:py123]" -# tox_proj = tox_project({"tox.ini": ini}) -# execute_calls = tox_proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) -# result = tox_proj.run("-e", "py123") -# result.assert_success() -# assert len(execute_calls) == 2 + outcome = proj.run("-e", "py123") + outcome.assert_success() -# venv = VirtualEnv(config.envconfigs["py123"]) -# assert venv.path == config.envconfigs["py123"].envdir + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 3 -# with mocksession.newaction(venv.name, "getenv") as action: -# tox_testenv_create(action=action, venv=venv) -# pcalls = mocksession._pcalls -# assert len(pcalls) >= 1 -# call = pcalls[-1] -# assert "conda" in call.args[0] -# assert "create" == call.args[1] -# assert "--yes" == call.args[2] -# assert "-p" == call.args[3] -# assert venv.path == call.args[4] -# assert call.args[5].startswith("python=") + create_env_cmd = executed_shell_commands[1] + assert "conda" in create_env_cmd[0] + assert "create" == create_env_cmd[1] + assert "-p" == create_env_cmd[2] + assert str(proj.path / ".tox" / "py123") == create_env_cmd[3] + assert create_env_cmd[4].startswith("python=") + assert "--yes" == create_env_cmd[5] + assert "--quiet" == create_env_cmd[6] # def create_test_env(config, mocksession, envname): From b2b45c30ad26ba8d67d5a62bdfaefcb5bb1f5580 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Thu, 1 Jun 2023 22:21:35 +0100 Subject: [PATCH 19/39] 2nd test ready. --- tests/test_conda_env.py | 93 ++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 035e796..c507bf2 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -51,65 +51,64 @@ def test_conda_create(tox_project, mock_conda_env_runner): assert "--yes" == create_env_cmd[5] assert "--quiet" == create_env_cmd[6] -# def create_test_env(config, mocksession, envname): +def test_install_deps_no_conda(tox_project, mock_conda_env_runner): + """Test installation using conda when no conda_deps are given""" + env_name = "py123" + ini = f""" + [testenv:{env_name}] + deps = + numpy + -r requirements.txt + astropy + """ + proj = tox_project({"tox.ini": ini}) + (proj.path / "requirements.txt").touch() -# venv = VirtualEnv(config.envconfigs[envname]) -# with mocksession.newaction(venv.name, "getenv") as action: -# tox_testenv_create(action=action, venv=venv) -# pcalls = mocksession._pcalls -# assert len(pcalls) >= 1 -# pcalls[:] = [] + outcome = proj.run("-e", "py123") + outcome.assert_success() -# return venv, action, pcalls + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 4 + cmd = executed_shell_commands[2] + cmd_conda_prefix = " ".join(cmd[:5]) + cmd_pip_install = " ".join(cmd[5:]) -# def test_install_deps_no_conda(newconfig, mocksession, monkeypatch): -# """Test installation using conda when no conda_deps are given""" -# # No longer remove the temporary script, so we can check its contents. -# monkeypatch.delattr(PopenInActivatedEnv, "__del__", raising=False) + assert f"conda run -p {str(proj.path / '.tox' / env_name)} --live-stream" in cmd_conda_prefix -# env_name = "py123" -# config = newconfig( -# [], -# """ -# [testenv:{}] -# deps= -# numpy -# -r requirements.txt -# astropy -# """.format( -# env_name -# ), -# ) + assert cmd_pip_install.startswith("python -I -m pip install") + assert "numpy" in cmd_pip_install + assert "astropy" in cmd_pip_install + assert "-r requirements.txt" in cmd_pip_install -# config.toxinidir.join("requirements.txt").write("") + # config.toxinidir.join("requirements.txt").write("") -# venv, action, pcalls = create_test_env(config, mocksession, env_name) + # venv, action, pcalls = create_test_env(config, mocksession, env_name) -# assert len(venv.envconfig.deps) == 3 -# assert len(venv.envconfig.conda_deps) == 0 + # assert len(venv.envconfig.deps) == 3 + # assert len(venv.envconfig.conda_deps) == 0 -# tox_testenv_install_deps(action=action, venv=venv) + # tox_testenv_install_deps(action=action, venv=venv) -# assert len(pcalls) >= 1 + # assert len(pcalls) >= 1 -# call = pcalls[-1] + # call = pcalls[-1] + + # if tox.INFO.IS_WIN: + # script_lines = " ".join(call.args).split(" && ", maxsplit=1) + # pattern = r"conda\.bat activate .*{}".format(re.escape(env_name)) + # else: + # # Get the cmd args from the script. + # shell, cmd_script = call.args + # assert shell == "/bin/sh" + # with open(cmd_script) as stream: + # script_lines = stream.readlines() + # pattern = r"eval \"\$\(/.*/conda shell\.posix activate /.*/{}\)\"".format(env_name) + + # assert re.match(pattern, script_lines[0]) -# if tox.INFO.IS_WIN: -# script_lines = " ".join(call.args).split(" && ", maxsplit=1) -# pattern = r"conda\.bat activate .*{}".format(re.escape(env_name)) -# else: -# # Get the cmd args from the script. -# shell, cmd_script = call.args -# assert shell == "/bin/sh" -# with open(cmd_script) as stream: -# script_lines = stream.readlines() -# pattern = r"eval \"\$\(/.*/conda shell\.posix activate /.*/{}\)\"".format(env_name) - -# assert re.match(pattern, script_lines[0]) - -# cmd = script_lines[1].split() -# assert cmd[-6:] == ["-m", "pip", "install", "numpy", "-rrequirements.txt", "astropy"] + # cmd = script_lines[1].split() + # assert cmd[-6:] == ["-m", "pip", "install", "numpy", "-rrequirements.txt", "astropy"] # def test_install_conda_deps(newconfig, mocksession): From 580330a8c3c5ef767639043f05021081e792c70e Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Thu, 1 Jun 2023 22:44:10 +0100 Subject: [PATCH 20/39] 4th test. --- tests/test_conda_env.py | 133 ++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 81 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index c507bf2..7ce3a4f 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -32,14 +32,17 @@ def test_conda_create(tox_project, mock_conda_env_runner): - ini = "[testenv:py123]" + ini = """ + [testenv:py123] + skip_install = True + """ proj = tox_project({"tox.ini": ini}) outcome = proj.run("-e", "py123") outcome.assert_success() executed_shell_commands = mock_conda_env_runner - assert len(executed_shell_commands) == 3 + assert len(executed_shell_commands) == 2 create_env_cmd = executed_shell_commands[1] @@ -52,10 +55,10 @@ def test_conda_create(tox_project, mock_conda_env_runner): assert "--quiet" == create_env_cmd[6] def test_install_deps_no_conda(tox_project, mock_conda_env_runner): - """Test installation using conda when no conda_deps are given""" env_name = "py123" ini = f""" [testenv:{env_name}] + skip_install = True deps = numpy -r requirements.txt @@ -68,107 +71,75 @@ def test_install_deps_no_conda(tox_project, mock_conda_env_runner): outcome.assert_success() executed_shell_commands = mock_conda_env_runner - assert len(executed_shell_commands) == 4 + assert len(executed_shell_commands) == 3 cmd = executed_shell_commands[2] cmd_conda_prefix = " ".join(cmd[:5]) cmd_pip_install = " ".join(cmd[5:]) - assert f"conda run -p {str(proj.path / '.tox' / env_name)} --live-stream" in cmd_conda_prefix + assert cmd_conda_prefix.endswith(f"conda run -p {str(proj.path / '.tox' / env_name)} --live-stream") assert cmd_pip_install.startswith("python -I -m pip install") assert "numpy" in cmd_pip_install assert "astropy" in cmd_pip_install assert "-r requirements.txt" in cmd_pip_install - # config.toxinidir.join("requirements.txt").write("") - - # venv, action, pcalls = create_test_env(config, mocksession, env_name) - - # assert len(venv.envconfig.deps) == 3 - # assert len(venv.envconfig.conda_deps) == 0 - - # tox_testenv_install_deps(action=action, venv=venv) - - # assert len(pcalls) >= 1 - - # call = pcalls[-1] - - # if tox.INFO.IS_WIN: - # script_lines = " ".join(call.args).split(" && ", maxsplit=1) - # pattern = r"conda\.bat activate .*{}".format(re.escape(env_name)) - # else: - # # Get the cmd args from the script. - # shell, cmd_script = call.args - # assert shell == "/bin/sh" - # with open(cmd_script) as stream: - # script_lines = stream.readlines() - # pattern = r"eval \"\$\(/.*/conda shell\.posix activate /.*/{}\)\"".format(env_name) - - # assert re.match(pattern, script_lines[0]) - - # cmd = script_lines[1].split() - # assert cmd[-6:] == ["-m", "pip", "install", "numpy", "-rrequirements.txt", "astropy"] +def test_install_conda_no_pip(tox_project, mock_conda_env_runner): + env_name = "py123" + ini = f""" + [testenv:{env_name}] + skip_install = True + conda_deps = + pytest + asdf + """ + proj = tox_project({"tox.ini": ini}) -# def test_install_conda_deps(newconfig, mocksession): -# config = newconfig( -# [], -# """ -# [testenv:py123] -# deps= -# numpy -# astropy -# conda_deps= -# pytest -# asdf -# """, -# ) + outcome = proj.run("-e", "py123") + outcome.assert_success() -# venv, action, pcalls = create_test_env(config, mocksession, "py123") + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 3 -# assert len(venv.envconfig.conda_deps) == 2 -# assert len(venv.envconfig.deps) == 2 + len(venv.envconfig.conda_deps) + cmd = executed_shell_commands[2] + cmd_conda_prefix = " ".join(cmd[:6]) + cmd_packages = " ".join(cmd[6:]) -# tox_testenv_install_deps(action=action, venv=venv) -# # We expect two calls: one for conda deps, and one for pip deps -# assert len(pcalls) >= 2 -# call = pcalls[-2] -# conda_cmd = call.args -# assert "conda" in os.path.split(conda_cmd[0])[-1] -# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] -# # Make sure that python is explicitly given as part of every conda install -# # in order to avoid inadvertent upgrades of python itself. -# assert conda_cmd[6].startswith("python=") -# assert conda_cmd[7:9] == ["pytest", "asdf"] + assert cmd_conda_prefix.endswith(f"conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}") + assert "asdf" in cmd_packages + assert "pytest" in cmd_packages + # Make sure that python is explicitly given as part of every conda install + # in order to avoid inadvertent upgrades of python itself. + assert "python=" in cmd_packages -# def test_install_conda_no_pip(newconfig, mocksession): -# config = newconfig( -# [], -# """ -# [testenv:py123] -# conda_deps= -# pytest -# asdf -# """, -# ) -# venv, action, pcalls = create_test_env(config, mocksession, "py123") +def test_install_conda_with_deps(tox_project, mock_conda_env_runner): + env_name = "py123" + ini = f""" + [testenv:{env_name}] + skip_install = True + deps = + numpy + astropy + conda_deps = + pytest + asdf + """ + proj = tox_project({"tox.ini": ini}) -# assert len(venv.envconfig.conda_deps) == 2 -# assert len(venv.envconfig.deps) == len(venv.envconfig.conda_deps) + outcome = proj.run("-e", "py123") + outcome.assert_success() -# tox_testenv_install_deps(action=action, venv=venv) -# # We expect only one call since there are no true pip dependencies -# assert len(pcalls) >= 1 + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 4 -# # Just a quick sanity check for the conda install command -# call = pcalls[-1] -# conda_cmd = call.args -# assert "conda" in os.path.split(conda_cmd[0])[-1] -# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] + cmd_conda = " ".join(executed_shell_commands[2]) + pip_cmd = " ".join(executed_shell_commands[3]) + assert "conda install --quiet --yes -p" in cmd_conda + assert "python -I -m pip install" in pip_cmd # def test_update(tmpdir, newconfig, mocksession): # pkg = tmpdir.ensure("package.tar.gz") From b5104ee3bca1af4fe75a04edb4ed823234f9c119 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Thu, 1 Jun 2023 22:46:28 +0100 Subject: [PATCH 21/39] Format. --- tests/conftest.py | 4 ++-- tests/test_conda_env.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5e6f628..7af5dcd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,6 +50,7 @@ def _init( return _init + @pytest.fixture def mock_conda_env_runner(request, monkeypatch): class MockExecuteStatus(ExecuteStatus): @@ -100,7 +101,7 @@ def cmd(self) -> Sequence[str]: return self.request.cmd shell_cmds = [] - no_mocked_run_ids = getattr(request, 'param', None) + no_mocked_run_ids = getattr(request, "param", None) if no_mocked_run_ids is None: no_mocked_run_ids = ["_get_python"] original_execute_instance_factor = CondaEnvRunner._execute_instance_factory @@ -118,4 +119,3 @@ def mock_execute_instance_factory( monkeypatch.setattr(CondaEnvRunner, "_execute_instance_factory", mock_execute_instance_factory) yield shell_cmds - diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 7ce3a4f..c3704ff 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -54,6 +54,7 @@ def test_conda_create(tox_project, mock_conda_env_runner): assert "--yes" == create_env_cmd[5] assert "--quiet" == create_env_cmd[6] + def test_install_deps_no_conda(tox_project, mock_conda_env_runner): env_name = "py123" ini = f""" @@ -77,7 +78,9 @@ def test_install_deps_no_conda(tox_project, mock_conda_env_runner): cmd_conda_prefix = " ".join(cmd[:5]) cmd_pip_install = " ".join(cmd[5:]) - assert cmd_conda_prefix.endswith(f"conda run -p {str(proj.path / '.tox' / env_name)} --live-stream") + assert cmd_conda_prefix.endswith( + f"conda run -p {str(proj.path / '.tox' / env_name)} --live-stream" + ) assert cmd_pip_install.startswith("python -I -m pip install") assert "numpy" in cmd_pip_install @@ -106,7 +109,9 @@ def test_install_conda_no_pip(tox_project, mock_conda_env_runner): cmd_conda_prefix = " ".join(cmd[:6]) cmd_packages = " ".join(cmd[6:]) - assert cmd_conda_prefix.endswith(f"conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}") + assert cmd_conda_prefix.endswith( + f"conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}" + ) assert "asdf" in cmd_packages assert "pytest" in cmd_packages @@ -141,6 +146,7 @@ def test_install_conda_with_deps(tox_project, mock_conda_env_runner): assert "conda install --quiet --yes -p" in cmd_conda assert "python -I -m pip install" in pip_cmd + # def test_update(tmpdir, newconfig, mocksession): # pkg = tmpdir.ensure("package.tar.gz") # config = newconfig( From e77fd71c9e9e5acd487e5801aa3f279dc48b5e99 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 10:58:41 +0100 Subject: [PATCH 22/39] conda_spec test added. --- tests/test_conda_env.py | 49 +++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index c3704ff..4e81ad4 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -147,30 +147,37 @@ def test_install_conda_with_deps(tox_project, mock_conda_env_runner): assert "python -I -m pip install" in pip_cmd -# def test_update(tmpdir, newconfig, mocksession): -# pkg = tmpdir.ensure("package.tar.gz") -# config = newconfig( -# [], -# """ -# [testenv:py123] -# deps= -# numpy -# astropy -# conda_deps= -# pytest -# asdf -# """, -# ) +def test_conda_spec(tox_project, mock_conda_env_runner): + env_name = "py123" + ini = f""" + [testenv:{env_name}] + skip_install = True + conda_deps = + numpy + astropy + conda_spec = conda_spec.txt + """ + proj = tox_project({"tox.ini": ini}) + (proj.path / "conda_spec.txt").touch() -# venv, action, pcalls = create_test_env(config, mocksession, "py123") -# tox_testenv_install_deps(action=action, venv=venv) + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 3 -# venv.hook.tox_testenv_create = tox_testenv_create -# venv.hook.tox_testenv_install_deps = tox_testenv_install_deps -# with mocksession.newaction(venv.name, "update") as action: -# venv.update(action) -# venv.installpkg(pkg, action) + cmd = executed_shell_commands[2] + cmd_conda_prefix = " ".join(cmd[:6]) + cmd_packages = " ".join(cmd[6:]) + + assert cmd_conda_prefix.endswith( + f"conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}" + ) + assert "astropy" in cmd_packages + assert "numpy" in cmd_packages + assert "python=" in cmd_packages + assert "--file=conda_spec.txt" in cmd_packages # def test_conda_spec(tmpdir, newconfig, mocksession): # """Test environment creation when conda_spec given""" From fcff0ccbbccbe3fa35e2d0156484cfec9e752c2d Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 12:05:47 +0100 Subject: [PATCH 23/39] Add conda-env test and fixes. --- tests/test_conda_env.py | 195 ++++++++++------------------------------ tox_conda/plugin.py | 9 +- 2 files changed, 52 insertions(+), 152 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 4e81ad4..eb06259 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -8,6 +8,7 @@ from typing import Any, Callable, Dict, Optional, Sequence, Union from unittest.mock import mock_open, patch +from fnmatch import fnmatch import pytest import tox import tox.run @@ -179,124 +180,59 @@ def test_conda_spec(tox_project, mock_conda_env_runner): assert "python=" in cmd_packages assert "--file=conda_spec.txt" in cmd_packages -# def test_conda_spec(tmpdir, newconfig, mocksession): -# """Test environment creation when conda_spec given""" -# txt = tmpdir.join("conda-spec.txt") -# txt.write( -# """ -# pytest -# """ -# ) -# config = newconfig( -# [], -# """ -# [testenv:py123] -# conda_deps= -# numpy -# astropy -# conda_spec={} -# """.format( -# str(txt) -# ), -# ) -# venv, action, pcalls = create_test_env(config, mocksession, "py123") - -# assert venv.envconfig.conda_spec -# assert len(venv.envconfig.conda_deps) == 2 - -# tox_testenv_install_deps(action=action, venv=venv) -# # We expect conda_spec to be appended to conda deps install -# assert len(pcalls) >= 1 -# call = pcalls[-1] -# conda_cmd = call.args -# assert "conda" in os.path.split(conda_cmd[0])[-1] -# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] -# # Make sure that python is explicitly given as part of every conda install -# # in order to avoid inadvertent upgrades of python itself. -# assert conda_cmd[6].startswith("python=") -# assert conda_cmd[7:9] == ["numpy", "astropy"] -# assert conda_cmd[-1].startswith("--file") -# assert conda_cmd[-1].endswith("conda-spec.txt") - - -# def test_empty_conda_spec_and_env(tmpdir, newconfig, mocksession): -# """Test environment creation when empty conda_spec and conda_env.""" -# txt = tmpdir.join("conda-spec.txt") -# txt.write( -# """ -# pytest -# """ -# ) -# config = newconfig( -# [], -# """ -# [testenv:py123] -# conda_env= -# foo: path-to.yml -# conda_spec= -# foo: path-to.yml -# """, -# ) -# venv, _, _ = create_test_env(config, mocksession, "py123") - -# assert venv.envconfig.conda_spec is None -# assert venv.envconfig.conda_env is None - - -# def test_conda_env(tmpdir, newconfig, mocksession): -# """Test environment creation when conda_env given""" -# yml = tmpdir.join("conda-env.yml") -# yml.write( -# """ -# name: tox-conda -# channels: -# - conda-forge -# - nodefaults -# dependencies: -# - numpy -# - astropy -# - pip: -# - pytest -# """ -# ) -# config = newconfig( -# [], -# """ -# [testenv:py123] -# conda_env={} -# """.format( -# str(yml) -# ), -# ) +def test_conda_env(tmp_path, tox_project, mock_conda_env_runner): + env_name = "py123" + ini = f""" + [testenv:{env_name}] + skip_install = True + conda_env = conda-env.yml + """ + yaml = """ + name: tox-conda + channels: + - conda-forge + - nodefaults + dependencies: + - numpy + - astropy + - pip: + - pytest + """ + proj = tox_project({"tox.ini": ini}) + (proj.path / "conda-env.yml").write_text(yaml) -# venv = VirtualEnv(config.envconfigs["py123"]) -# assert venv.path == config.envconfigs["py123"].envdir + mock_file = mock_open() + + mock_temp_file = tmp_path / "mock_temp_file.yml" + def open_mock_temp_file(*args, **kwargs): + return mock_temp_file.open("w") -# venv, action, pcalls = create_test_env(config, mocksession, "py123") -# assert venv.envconfig.conda_env + with patch("tox_conda.plugin.tempfile.NamedTemporaryFile", open_mock_temp_file): + with patch.object(pathlib.Path, "unlink", autospec=True) as mock_unlink: + outcome = proj.run("-e", "py123") + outcome.assert_success() -# mock_file = mock_open() -# with patch("tox_conda.plugin.tempfile.NamedTemporaryFile", mock_file): -# with patch.object(pathlib.Path, "unlink", autospec=True) as mock_unlink: -# with mocksession.newaction(venv.name, "getenv") as action: -# tox_testenv_create(action=action, venv=venv) -# mock_unlink.assert_called_once + mock_unlink.assert_called_once + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 2 -# mock_file.assert_called_with(dir=tmpdir, prefix="tox_conda_tmp", suffix=".yaml", delete=False) + create_env_cmd = executed_shell_commands[1] -# pcalls = mocksession._pcalls -# assert len(pcalls) >= 1 -# call = pcalls[-1] -# cmd = call.args -# assert "conda" in os.path.split(cmd[0])[-1] -# assert cmd[1:4] == ["env", "create", "-p"] -# assert venv.path == call.args[4] -# assert call.args[5].startswith("--file") -# assert cmd[6] == str(mock_file().name) + assert "conda" in create_env_cmd[0] + assert "env" == create_env_cmd[1] + assert "create" == create_env_cmd[2] + assert "-p" == create_env_cmd[3] + assert str(proj.path / ".tox" / "py123") == create_env_cmd[4] + assert "--file" == create_env_cmd[5] + assert str(mock_temp_file) == create_env_cmd[6] + assert "--quiet" == create_env_cmd[7] + assert "--force" == create_env_cmd[8] -# yaml = YAML() -# tmp_env = yaml.load(mock_open_to_string(mock_file)) -# assert tmp_env["dependencies"][-1].startswith("python=") + # Check that the temporary file has the correct contents + yaml = YAML() + tmp_env = yaml.load(mock_temp_file) + assert tmp_env["dependencies"][-1].startswith("python=") # def test_conda_env_and_spec(tmpdir, newconfig, mocksession): @@ -420,38 +356,3 @@ def test_conda_spec(tox_project, mock_conda_env_runner): # assert venv.path == call.args[4] # assert call.args[5] == "--override-channels" # assert call.args[6].startswith("python=") - - -# def test_verbosity(newconfig, mocksession): -# config = newconfig( -# [], -# """ -# [testenv:py1] -# conda_deps=numpy -# [testenv:py2] -# conda_deps=numpy -# """, -# ) - -# venv, action, pcalls = create_test_env(config, mocksession, "py1") -# tox_testenv_install_deps(action=action, venv=venv) -# assert len(pcalls) == 1 -# call = pcalls[0] -# assert "conda" in call.args[0] -# assert "install" == call.args[1] -# assert isinstance(call.stdout, io.IOBase) - -# tox.reporter.update_default_reporter( -# tox.reporter.Verbosity.DEFAULT, tox.reporter.Verbosity.DEBUG -# ) -# venv, action, pcalls = create_test_env(config, mocksession, "py2") -# tox_testenv_install_deps(action=action, venv=venv) -# assert len(pcalls) == 1 -# call = pcalls[0] -# assert "conda" in call.args[0] -# assert "install" == call.args[1] -# assert not isinstance(call.stdout, io.IOBase) - - -# def mock_open_to_string(mock): -# return "".join(call.args[0] for call in mock().write.call_args_list) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 76f250c..7ab3dab 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -127,10 +127,9 @@ def python_cache(self) -> Dict[str, Any]: conda_dict["env_spec"] = "-n" conda_dict["env"] = conda_name elif self.conf["conda_env"]: - conda_dict["env_spec"] = "-n" + conda_dict["env_spec"] = "-p" + conda_dict["env"] = str(self.env_dir) env_path = Path(self.conf["conda_env"]).resolve() - env_file = YAML().load(env_path) - conda_dict["env"] = env_file["name"] conda_dict["env_path"] = str(env_path) conda_dict["env_hash"] = hash_file(Path(self.conf["conda_env"]).resolve()) else: @@ -206,13 +205,13 @@ def _generate_env_create_command( tmp_env_file = tempfile.NamedTemporaryFile( dir=env_path.parent, prefix="tox_conda_tmp", - suffix=".yaml", + suffix=env_path.suffix, delete=False, ) yaml.dump(env_file, tmp_env_file) tmp_env_file.close() - cmd = f"'{conda_exe}' env create --file '{tmp_env_file.name}' --quiet --force" + cmd = f"'{conda_exe}' env create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}' --file '{tmp_env_file.name}' --quiet --force" def tear_down(): return Path(tmp_env_file.name).unlink() From 7bdd51ecb71a96e00b45be3fa004276eadc5b45b Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 12:13:14 +0100 Subject: [PATCH 24/39] Test for env. and spec. --- tests/test_conda_env.py | 105 ++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 70 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index eb06259..3d84bfc 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -200,8 +200,6 @@ def test_conda_env(tmp_path, tox_project, mock_conda_env_runner): """ proj = tox_project({"tox.ini": ini}) (proj.path / "conda-env.yml").write_text(yaml) - - mock_file = mock_open() mock_temp_file = tmp_path / "mock_temp_file.yml" def open_mock_temp_file(*args, **kwargs): @@ -235,78 +233,45 @@ def open_mock_temp_file(*args, **kwargs): assert tmp_env["dependencies"][-1].startswith("python=") -# def test_conda_env_and_spec(tmpdir, newconfig, mocksession): -# """Test environment creation when conda_env and conda_spec are given""" -# yml = tmpdir.join("conda-env.yml") -# yml.write( -# """ -# name: tox-conda -# channels: -# - conda-forge -# - nodefaults -# dependencies: -# - numpy -# - astropy -# """ -# ) -# txt = tmpdir.join("conda-spec.txt") -# txt.write( -# """ -# pytest -# """ -# ) -# config = newconfig( -# [], -# """ -# [testenv:py123] -# conda_env={} -# conda_spec={} -# """.format( -# str(yml), str(txt) -# ), -# ) -# venv, action, pcalls = create_test_env(config, mocksession, "py123") - -# assert venv.envconfig.conda_env -# assert venv.envconfig.conda_spec - -# mock_file = mock_open() -# with patch("tox_conda.plugin.tempfile.NamedTemporaryFile", mock_file): -# with patch.object(pathlib.Path, "unlink", autospec=True) as mock_unlink: -# with mocksession.newaction(venv.name, "getenv") as action: -# tox_testenv_create(action=action, venv=venv) -# mock_unlink.assert_called_once +def test_conda_env_and_spec(tmp_path, tox_project, mock_conda_env_runner): + env_name = "py123" + ini = f""" + [testenv:{env_name}] + skip_install = True + conda_env = conda-env.yml + conda_spec = conda_spec.txt + """ + yaml = """ + name: tox-conda + channels: + - conda-forge + - nodefaults + dependencies: + - numpy + - astropy + - pip: + - pytest + """ + proj = tox_project({"tox.ini": ini}) + (proj.path / "conda-env.yml").write_text(yaml) + (proj.path / "conda_spec.txt").touch() + + outcome = proj.run("-e", "py123") + outcome.assert_success() -# mock_file.assert_called_with(dir=tmpdir, prefix="tox_conda_tmp", suffix=".yaml", delete=False) + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 3 -# pcalls = mocksession._pcalls -# assert len(pcalls) >= 1 -# call = pcalls[-1] -# cmd = call.args -# assert "conda" in os.path.split(cmd[0])[-1] -# assert cmd[1:4] == ["env", "create", "-p"] -# assert venv.path == call.args[4] -# assert call.args[5].startswith("--file") -# assert cmd[6] == str(mock_file().name) + create_env_cmd = executed_shell_commands[1] + install_cmd = executed_shell_commands[2] -# yaml = YAML() -# tmp_env = yaml.load(mock_open_to_string(mock_file)) -# assert tmp_env["dependencies"][-1].startswith("python=") + assert "conda" in create_env_cmd[0] + assert "env" == create_env_cmd[1] + assert "create" == create_env_cmd[2] -# with mocksession.newaction(venv.name, "getenv") as action: -# tox_testenv_install_deps(action=action, venv=venv) -# pcalls = mocksession._pcalls -# # We expect conda_spec to be appended to conda deps install -# assert len(pcalls) >= 1 -# call = pcalls[-1] -# conda_cmd = call.args -# assert "conda" in os.path.split(conda_cmd[0])[-1] -# assert conda_cmd[1:6] == ["install", "--quiet", "--yes", "-p", venv.path] -# # Make sure that python is explicitly given as part of every conda install -# # in order to avoid inadvertent upgrades of python itself. -# assert conda_cmd[6].startswith("python=") -# assert conda_cmd[-1].startswith("--file") -# assert conda_cmd[-1].endswith("conda-spec.txt") + assert "conda" in install_cmd[0] + assert "install" == install_cmd[1] + assert "--file=conda_spec.txt" in install_cmd # def test_conda_install_args(newconfig, mocksession): From 311a14e9b820877fe1a4e0f5e8e21a52caa44370 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 12:17:42 +0100 Subject: [PATCH 25/39] Add conda env. tests are ready. --- tests/test_conda_env.py | 94 ++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 3d84bfc..7eb8fb2 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -233,7 +233,7 @@ def open_mock_temp_file(*args, **kwargs): assert tmp_env["dependencies"][-1].startswith("python=") -def test_conda_env_and_spec(tmp_path, tox_project, mock_conda_env_runner): +def test_conda_env_and_spec(tox_project, mock_conda_env_runner): env_name = "py123" ini = f""" [testenv:{env_name}] @@ -274,50 +274,48 @@ def test_conda_env_and_spec(tmp_path, tox_project, mock_conda_env_runner): assert "--file=conda_spec.txt" in install_cmd -# def test_conda_install_args(newconfig, mocksession): -# config = newconfig( -# [], -# """ -# [testenv:py123] -# conda_deps= -# numpy -# conda_install_args= -# --override-channels -# """, -# ) - -# venv, action, pcalls = create_test_env(config, mocksession, "py123") - -# assert len(venv.envconfig.conda_install_args) == 1 - -# tox_testenv_install_deps(action=action, venv=venv) - -# call = pcalls[-1] -# assert call.args[6] == "--override-channels" - - -# def test_conda_create_args(newconfig, mocksession): -# config = newconfig( -# [], -# """ -# [testenv:py123] -# conda_create_args= -# --override-channels -# """, -# ) - -# venv = VirtualEnv(config.envconfigs["py123"]) -# assert venv.path == config.envconfigs["py123"].envdir - -# with mocksession.newaction(venv.name, "getenv") as action: -# tox_testenv_create(action=action, venv=venv) -# pcalls = mocksession._pcalls -# assert len(pcalls) >= 1 -# call = pcalls[-1] -# assert "conda" in call.args[0] -# assert "create" == call.args[1] -# assert "--yes" == call.args[2] -# assert "-p" == call.args[3] -# assert venv.path == call.args[4] -# assert call.args[5] == "--override-channels" -# assert call.args[6].startswith("python=") +def test_conda_install_args(tmp_path, tox_project, mock_conda_env_runner): + env_name = "py123" + ini = f""" + [testenv:{env_name}] + skip_install = True + conda_deps= + numpy + conda_install_args = --override-channels + """ + + proj = tox_project({"tox.ini": ini}) + + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 3 + + install_cmd = executed_shell_commands[2] + + assert "conda" in install_cmd[0] + assert "install" == install_cmd[1] + assert "--override-channels" in install_cmd + +def test_conda_create_args(tmp_path, tox_project, mock_conda_env_runner): + env_name = "py123" + ini = f""" + [testenv:{env_name}] + skip_install = True + conda_create_args = --override-channels + """ + + proj = tox_project({"tox.ini": ini}) + + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 2 + + create_cmd = executed_shell_commands[1] + + assert "conda" in create_cmd[0] + assert "create" == create_cmd[1] + assert "--override-channels" in create_cmd From af770cff129e3b263f7bdd44590be0e4bcbd254c Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 12:20:12 +0100 Subject: [PATCH 26/39] Add conda_name test. --- tests/test_conda_env.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 7eb8fb2..67c6cfa 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -55,6 +55,30 @@ def test_conda_create(tox_project, mock_conda_env_runner): assert "--yes" == create_env_cmd[5] assert "--quiet" == create_env_cmd[6] +def test_conda_create_with_name(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_name = myenv + """ + proj = tox_project({"tox.ini": ini}) + + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 2 + + create_env_cmd = executed_shell_commands[1] + + assert "conda" in create_env_cmd[0] + assert "create" == create_env_cmd[1] + assert "-n" == create_env_cmd[2] + assert "myenv" == create_env_cmd[3] + assert create_env_cmd[4].startswith("python=") + assert "--yes" == create_env_cmd[5] + assert "--quiet" == create_env_cmd[6] + def test_install_deps_no_conda(tox_project, mock_conda_env_runner): env_name = "py123" @@ -274,7 +298,7 @@ def test_conda_env_and_spec(tox_project, mock_conda_env_runner): assert "--file=conda_spec.txt" in install_cmd -def test_conda_install_args(tmp_path, tox_project, mock_conda_env_runner): +def test_conda_install_args(tox_project, mock_conda_env_runner): env_name = "py123" ini = f""" [testenv:{env_name}] @@ -298,7 +322,7 @@ def test_conda_install_args(tmp_path, tox_project, mock_conda_env_runner): assert "install" == install_cmd[1] assert "--override-channels" in install_cmd -def test_conda_create_args(tmp_path, tox_project, mock_conda_env_runner): +def test_conda_create_args(tox_project, mock_conda_env_runner): env_name = "py123" ini = f""" [testenv:{env_name}] @@ -319,3 +343,4 @@ def test_conda_create_args(tmp_path, tox_project, mock_conda_env_runner): assert "conda" in create_cmd[0] assert "create" == create_cmd[1] assert "--override-channels" in create_cmd + From 10868ef6140d0f66715a46cd452a3955ba7bbdf1 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 12:21:28 +0100 Subject: [PATCH 27/39] Format. --- tests/test_conda_env.py | 47 ++++++++++------------------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 67c6cfa..a4a3889 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -1,35 +1,7 @@ -import io -import os import pathlib -import re -import subprocess -from pathlib import Path -from types import ModuleType, TracebackType -from typing import Any, Callable, Dict, Optional, Sequence, Union -from unittest.mock import mock_open, patch - -from fnmatch import fnmatch -import pytest -import tox -import tox.run -from pytest import MonkeyPatch -from pytest_mock import MockerFixture +from unittest.mock import patch + from ruamel.yaml import YAML -from tox.config.sets import EnvConfigSet -from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteStatus, Outcome -from tox.execute.request import ExecuteRequest, shell_cmd -from tox.execute.stream import SyncWrite -from tox.plugin import manager -from tox.pytest import CaptureFixture, ToxProject, ToxProjectCreator -from tox.report import LOGGER, OutErr -from tox.run import run as tox_run -from tox.run import setup_state as previous_setup_state -from tox.session.cmd.run.parallel import ENV_VAR_KEY -from tox.session.state import State -from tox.tox_env import api as tox_env_api -from tox.tox_env.api import ToxEnv - -from tox_conda.plugin import CondaEnvRunner def test_conda_create(tox_project, mock_conda_env_runner): @@ -55,6 +27,7 @@ def test_conda_create(tox_project, mock_conda_env_runner): assert "--yes" == create_env_cmd[5] assert "--quiet" == create_env_cmd[6] + def test_conda_create_with_name(tox_project, mock_conda_env_runner): ini = """ [testenv:py123] @@ -204,6 +177,7 @@ def test_conda_spec(tox_project, mock_conda_env_runner): assert "python=" in cmd_packages assert "--file=conda_spec.txt" in cmd_packages + def test_conda_env(tmp_path, tox_project, mock_conda_env_runner): env_name = "py123" ini = f""" @@ -224,8 +198,9 @@ def test_conda_env(tmp_path, tox_project, mock_conda_env_runner): """ proj = tox_project({"tox.ini": ini}) (proj.path / "conda-env.yml").write_text(yaml) - + mock_temp_file = tmp_path / "mock_temp_file.yml" + def open_mock_temp_file(*args, **kwargs): return mock_temp_file.open("w") @@ -235,7 +210,7 @@ def open_mock_temp_file(*args, **kwargs): outcome.assert_success() mock_unlink.assert_called_once - + executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 @@ -279,7 +254,7 @@ def test_conda_env_and_spec(tox_project, mock_conda_env_runner): proj = tox_project({"tox.ini": ini}) (proj.path / "conda-env.yml").write_text(yaml) (proj.path / "conda_spec.txt").touch() - + outcome = proj.run("-e", "py123") outcome.assert_success() @@ -309,7 +284,7 @@ def test_conda_install_args(tox_project, mock_conda_env_runner): """ proj = tox_project({"tox.ini": ini}) - + outcome = proj.run("-e", "py123") outcome.assert_success() @@ -322,6 +297,7 @@ def test_conda_install_args(tox_project, mock_conda_env_runner): assert "install" == install_cmd[1] assert "--override-channels" in install_cmd + def test_conda_create_args(tox_project, mock_conda_env_runner): env_name = "py123" ini = f""" @@ -331,7 +307,7 @@ def test_conda_create_args(tox_project, mock_conda_env_runner): """ proj = tox_project({"tox.ini": ini}) - + outcome = proj.run("-e", "py123") outcome.assert_success() @@ -343,4 +319,3 @@ def test_conda_create_args(tox_project, mock_conda_env_runner): assert "conda" in create_cmd[0] assert "create" == create_cmd[1] assert "--override-channels" in create_cmd - From 3b7b57ee2486678bd4a81b73fff40839717cd385 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 16:37:57 +0100 Subject: [PATCH 28/39] Add cache tests. --- tests/test_cache.py | 231 +++++++++++++++++++++++++++++++++++++++++++ tests/test_config.py | 127 ------------------------ tox_conda/plugin.py | 4 +- 3 files changed, 233 insertions(+), 129 deletions(-) create mode 100644 tests/test_cache.py delete mode 100644 tests/test_config.py diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..125e196 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,231 @@ +import pathlib +from unittest.mock import patch + +from ruamel.yaml import YAML + +def assert_create_command(cmd): + assert "conda" in cmd[0] and ("create" == cmd[1] or "create" == cmd[2]) + +def assert_install_command(cmd): + assert "conda" in cmd[0] and "install" == cmd[1] + +def test_conda_no_recreate(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_env = conda-env.yml + conda_deps = + asdf + """ + yaml = """ + name: tox-conda + channels: + - conda-forge + - nodefaults + dependencies: + - numpy + - astropy + - pip: + - pytest + """ + for i in range(2): + proj = tox_project({"tox.ini": ini}) + (proj.path / "conda-env.yml").write_text(yaml) + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + # get_python, create env, install deps, get_python, and nothing else because no changes + assert len(executed_shell_commands) == 4 + assert_create_command(executed_shell_commands[1]) + assert_install_command(executed_shell_commands[2]) + +def test_conda_recreate_by_dependency_change(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_deps = + asdf + """ + ini_modified = """ + [testenv:py123] + skip_install = True + conda_deps = + asdf + black + """ + outcome = tox_project({"tox.ini": ini}).run("-e", "py123") + outcome.assert_success() + + outcome = tox_project({"tox.ini": ini_modified}).run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + # get_python, create env, install deps, get_python, create env, install deps + assert len(executed_shell_commands) == 6 + + assert_create_command(executed_shell_commands[1]) + assert_install_command(executed_shell_commands[2]) + assert_create_command(executed_shell_commands[4]) + assert_install_command(executed_shell_commands[5]) + + +def test_conda_recreate_by_env_file_path_change(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_env = conda-env-1.yml + """ + ini_modified = """ + [testenv:py123] + skip_install = True + conda_env = conda-env-2.yml + """ + yaml = """ + name: tox-conda + channels: + - conda-forge + - nodefaults + dependencies: + - numpy + - astropy + - pip: + - pytest + """ + + proj_1 = tox_project({"tox.ini": ini}) + (proj_1.path / "conda-env-1.yml").write_text(yaml) + outcome = proj_1.run("-e", "py123") + outcome.assert_success() + + proj_2 = tox_project({"tox.ini": ini_modified}) + (proj_2.path / "conda-env-2.yml").write_text(yaml) + outcome = proj_2.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + # get_python, create env, get_python, create env + assert len(executed_shell_commands) == 4 + + assert_create_command(executed_shell_commands[1]) + assert_create_command(executed_shell_commands[3]) + + +def test_conda_recreate_by_env_file_content_change(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_env = conda-env.yml + """ + yaml = """ + name: tox-conda + channels: + - conda-forge + - nodefaults + dependencies: + - numpy + - astropy + """ + yaml_modified = """ + name: tox-conda + channels: + - conda-forge + - nodefaults + dependencies: + - numpy + - astropy + - pip: + - pytest + """ + + proj = tox_project({"tox.ini": ini}) + (proj.path / "conda-env.yml").write_text(yaml) + outcome = proj.run("-e", "py123") + outcome.assert_success() + + (proj.path / "conda-env.yml").write_text(yaml_modified) + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + # get_python, create env, get_python, create env + assert len(executed_shell_commands) == 4 + + assert_create_command(executed_shell_commands[1]) + assert_create_command(executed_shell_commands[3]) + + +def test_conda_recreate_by_spec_file_path_change(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_spec = conda_spec-1.txt + """ + ini_modified = """ + [testenv:py123] + skip_install = True + conda_spec = conda_spec-2.txt + """ + yaml = """ + name: tox-conda + channels: + - conda-forge + - nodefaults + dependencies: + - numpy + - astropy + - pip: + - pytest + """ + + proj_1 = tox_project({"tox.ini": ini}) + (proj_1.path / "conda_spec-1.txt").touch() + outcome = proj_1.run("-e", "py123") + outcome.assert_success() + + proj_2 = tox_project({"tox.ini": ini_modified}) + (proj_2.path / "conda_spec-2.txt").touch() + outcome = proj_2.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + # get_python, create env, install deps, get_python, create env, install deps + assert len(executed_shell_commands) == 6 + + assert_create_command(executed_shell_commands[1]) + assert_install_command(executed_shell_commands[2]) + assert_create_command(executed_shell_commands[4]) + assert_install_command(executed_shell_commands[5]) + + +def test_conda_recreate_by_spec_file_content_change(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_spec = conda_spec.txt + """ + txt = """ + black + """ + txt_modified = """ + black + numpy + """ + + proj = tox_project({"tox.ini": ini}) + (proj.path / "conda_spec.txt").write_text(txt) + outcome = proj.run("-e", "py123") + outcome.assert_success() + + (proj.path / "conda_spec.txt").write_text(txt_modified) + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + # get_python, create env, install deps, get_python, create env, install deps + assert len(executed_shell_commands) == 6 + + assert_create_command(executed_shell_commands[1]) + assert_install_command(executed_shell_commands[2]) + assert_create_command(executed_shell_commands[4]) + assert_install_command(executed_shell_commands[5]) diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index b316441..0000000 --- a/tests/test_config.py +++ /dev/null @@ -1,127 +0,0 @@ -def test_conda_deps(tmpdir, newconfig): - config = newconfig( - [], - """ - [tox] - toxworkdir = {} - [testenv:py1] - deps= - hello - conda_deps= - world - something - """.format( - tmpdir - ), - ) - - assert len(config.envconfigs) == 1 - assert hasattr(config.envconfigs["py1"], "deps") - assert hasattr(config.envconfigs["py1"], "conda_deps") - assert len(config.envconfigs["py1"].conda_deps) == 2 - # For now, as a workaround, we temporarily add all conda dependencies to - # deps as well. This allows tox to know whether an environment needs to be - # updated or not. Eventually there may be a cleaner solution. - assert len(config.envconfigs["py1"].deps) == 3 - assert "world" == config.envconfigs["py1"].conda_deps[0].name - assert "something" == config.envconfigs["py1"].conda_deps[1].name - - -def test_conda_env_and_spec(tmpdir, newconfig): - config = newconfig( - [], - """ - [tox] - toxworkdir = {} - [testenv:py1] - conda_env = conda_env.yaml - conda_spec = conda_spec.txt - """.format( - tmpdir - ), - ) - - assert len(config.envconfigs) == 1 - assert config.envconfigs["py1"].conda_env == tmpdir / "conda_env.yaml" - assert config.envconfigs["py1"].conda_spec == tmpdir / "conda_spec.txt" - # Conda env and spec files get added to deps to allow tox to detect changes. - # Similar to conda_deps in the test above. - assert hasattr(config.envconfigs["py1"], "deps") - assert len(config.envconfigs["py1"].deps) == 2 - assert any(dep.name == tmpdir / "conda_env.yaml" for dep in config.envconfigs["py1"].deps) - assert any(dep.name == tmpdir / "conda_spec.txt" for dep in config.envconfigs["py1"].deps) - - -def test_no_conda_deps(tmpdir, newconfig): - config = newconfig( - [], - """ - [tox] - toxworkdir = {} - [testenv:py1] - deps= - hello - """.format( - tmpdir - ), - ) - - assert len(config.envconfigs) == 1 - assert hasattr(config.envconfigs["py1"], "deps") - assert hasattr(config.envconfigs["py1"], "conda_deps") - assert hasattr(config.envconfigs["py1"], "conda_channels") - assert len(config.envconfigs["py1"].conda_deps) == 0 - assert len(config.envconfigs["py1"].conda_channels) == 0 - assert len(config.envconfigs["py1"].deps) == 1 - - -def test_conda_channels(tmpdir, newconfig): - config = newconfig( - [], - """ - [tox] - toxworkdir = {} - [testenv:py1] - deps= - hello - conda_deps= - something - else - conda_channels= - conda-forge - """.format( - tmpdir - ), - ) - - assert len(config.envconfigs) == 1 - assert hasattr(config.envconfigs["py1"], "deps") - assert hasattr(config.envconfigs["py1"], "conda_deps") - assert hasattr(config.envconfigs["py1"], "conda_channels") - assert len(config.envconfigs["py1"].conda_channels) == 1 - assert "conda-forge" in config.envconfigs["py1"].conda_channels - - -def test_conda_force_deps(tmpdir, newconfig): - config = newconfig( - ["--force-dep=something<42.1"], - """ - [tox] - toxworkdir = {} - [testenv:py1] - deps= - hello - conda_deps= - something - else - conda_channels= - conda-forge - """.format( - tmpdir - ), - ) - - assert len(config.envconfigs) == 1 - assert hasattr(config.envconfigs["py1"], "conda_deps") - assert len(config.envconfigs["py1"].conda_deps) == 2 - assert "something<42.1" == config.envconfigs["py1"].conda_deps[0].name diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 7ab3dab..2d9b9e8 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -32,12 +32,12 @@ class CondaEnvRunner(PythonRun): # Can be overridden in tests @staticmethod - def _execute_instance_factory( + def _default_execute_instance_factory( request: ExecuteRequest, options: ExecuteOptions, out: SyncWrite, err: SyncWrite ): return LocalSubProcessExecuteInstance(request, options, out, err) - _execute_instance_factory: Union[ExecuteInstance, Callable] = _execute_instance_factory + _execute_instance_factory: Callable = _default_execute_instance_factory def __init__(self, create_args: ToxEnvCreateArgs) -> None: self._installer = None From 8e863c41c5203cc081a77c1b688185a8decd558b Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 18:06:49 +0100 Subject: [PATCH 29/39] Add more tests and fixes. --- tests/conftest.py | 2 +- tests/test_cache.py | 9 ++- tests/test_conda.py | 145 +++++++++++----------------------------- tests/test_conda_env.py | 111 ++++++++---------------------- 4 files changed, 74 insertions(+), 193 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7af5dcd..16757d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,7 +109,7 @@ def cmd(self) -> Sequence[str]: def mock_execute_instance_factory( request: ExecuteRequest, options: ExecuteOptions, out: SyncWrite, err: SyncWrite ): - shell_cmds.append(request.cmd) + shell_cmds.append(request.shell_cmd) if request.run_id not in no_mocked_run_ids: return MockExecuteInstance(request, options, out, err, 0) diff --git a/tests/test_cache.py b/tests/test_cache.py index 125e196..93e362d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,13 +1,12 @@ -import pathlib -from unittest.mock import patch +"""Cache tests.""" -from ruamel.yaml import YAML +from fnmatch import fnmatch def assert_create_command(cmd): - assert "conda" in cmd[0] and ("create" == cmd[1] or "create" == cmd[2]) + assert fnmatch(cmd, "*conda create*") or fnmatch(cmd, "*conda env create*") def assert_install_command(cmd): - assert "conda" in cmd[0] and "install" == cmd[1] + assert fnmatch(cmd, "*conda install*") def test_conda_no_recreate(tox_project, mock_conda_env_runner): ini = """ diff --git a/tests/test_conda.py b/tests/test_conda.py index e0143e1..a1f95fb 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -1,10 +1,18 @@ +import os import shutil +import pytest import tox +from tox.tox_env.errors import Fail import tox_conda.plugin +from fnmatch import fnmatch + +def assert_conda_context(proj, env_name, shell_command, expected_command): + assert fnmatch(shell_command, f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream {expected_command}") + def test_conda(cmd, initproj): # The path has a blank space on purpose for testing issue #119. initproj( @@ -33,121 +41,48 @@ def index_of(m): assert result.outlines[-4] == "True" -def test_conda_run_command(cmd, initproj): - """Check that all the commands are run from an activated anaconda env. - - This is done by looking at the CONDA_PREFIX environment variable which contains - the environment name. - This variable is dumped to a file because commands_{pre,post} do not redirect - their outputs. +def test_conda_run_command(tox_project, mock_conda_env_runner): + """Check that all the commands are run from an activated anaconda env.""" + ini = """ + [testenv:py123] + skip_install = True + commands_pre = python --version + commands = pytest + commands_post = black """ - env_name = "foobar" - initproj( - "pkg-1", - filedefs={ - "tox.ini": """ - [tox] - skipsdist=True - [testenv:{}] - deps = - pip >0,<999 - -r requirements.txt - commands_pre = python -c "import os; open('commands_pre', 'w').write(os.environ['CONDA_PREFIX'])" - commands = python -c "import os; open('commands', 'w').write(os.environ['CONDA_PREFIX'])" - commands_post = python -c "import os; open('commands_post', 'w').write(os.environ['CONDA_PREFIX'])" - """.format( # noqa: E501 - env_name - ), - "requirements.txt": "", - }, - ) + proj = tox_project({"tox.ini": ini}) + outcome = proj.run("-e", "py123") + outcome.assert_success() - result = cmd("-v", "-e", env_name) - result.assert_success() - - for filename in ("commands_pre", "commands_post", "commands"): - assert open(filename).read().endswith(env_name) - - # Run once again when the env creation hooks are not called. - result = cmd("-v", "-e", env_name) - result.assert_success() - - for filename in ("commands_pre", "commands_post", "commands"): - assert open(filename).read().endswith(env_name) + executed_shell_commands = mock_conda_env_runner + + assert len(executed_shell_commands) == 5 + assert_conda_context(proj, "py123", executed_shell_commands[2], "python --version") + assert_conda_context(proj, "py123", executed_shell_commands[3], "pytest") + assert_conda_context(proj, "py123", executed_shell_commands[4], "black") -def test_missing_conda(cmd, initproj, monkeypatch): +def test_missing_conda(tox_project, mock_conda_env_runner, monkeypatch): """Check that an error is shown when the conda executable is not found.""" - - initproj( - "pkg-1", - filedefs={ - "tox.ini": """ - [tox] - require = tox-conda - """, - }, - ) - + ini = """ + [tox] + require = tox-conda + [testenv:py123] + skip_install = True + """ # Prevent conda from being found. original_which = shutil.which - def which(path): # pragma: no cover - if path.endswith("conda"): + def which(cmd, mode=os.F_OK | os.X_OK, path=None): + if cmd.endswith("conda"): return None - return original_which(path) + return original_which(cmd, mode, path) monkeypatch.setattr(shutil, "which", which) + monkeypatch.delenv("_CONDA_EXE", raising=False) + monkeypatch.delenv("CONDA_EXE", raising=False) - result = cmd() - - assert result.outlines == ["ERROR: {}".format(tox_conda.plugin.MISSING_CONDA_ERROR)] - - -def test_issue_115(cmd, initproj): - """Verify that a conda activation script is sourced. - - https://docs.conda.io/projects/conda-build/en/latest/resources/activate-scripts.html - """ - if tox.INFO.IS_WIN: - build_script_name = "build.bat" - build_script = """ - setlocal EnableDelayedExpansion - mkdir %CONDA_PREFIX%\\etc\\conda\\activate.d - copy activate.bat %CONDA_PREFIX%\\etc\\conda\\activate.d - """ - activate_script_name = "activate.bat" - activate_script = """ - set DUMMY=0 - """ - commands_pre = "build.bat" - - else: - build_script_name = "build.sh" - build_script = """ - mkdir -p "${CONDA_PREFIX}/etc/conda/activate.d" - cp activate.sh "${CONDA_PREFIX}/etc/conda/activate.d" - """ - activate_script_name = "activate.sh" - activate_script = """ - export DUMMY=0 - """ - commands_pre = "/bin/sh build.sh" + with pytest.raises(Fail) as exc_info: + tox_project({"tox.ini": ini}).run("-e", "py123") - initproj( - "115", - filedefs={ - build_script_name: build_script, - activate_script_name: activate_script, - "tox.ini": """ - [testenv] - commands_pre = {} - commands = python -c "import os; assert 'DUMMY' in os.environ" - """.format( - commands_pre - ), - }, - ) - - result = cmd() - result.assert_success() + assert str(exc_info.value) == "Failed to find 'conda' executable." diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index a4a3889..319a88d 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -1,8 +1,12 @@ +"""Conda environment creation and installation tests.""" + import pathlib from unittest.mock import patch from ruamel.yaml import YAML +from fnmatch import fnmatch + def test_conda_create(tox_project, mock_conda_env_runner): ini = """ @@ -16,16 +20,7 @@ def test_conda_create(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 - - create_env_cmd = executed_shell_commands[1] - - assert "conda" in create_env_cmd[0] - assert "create" == create_env_cmd[1] - assert "-p" == create_env_cmd[2] - assert str(proj.path / ".tox" / "py123") == create_env_cmd[3] - assert create_env_cmd[4].startswith("python=") - assert "--yes" == create_env_cmd[5] - assert "--quiet" == create_env_cmd[6] + assert fnmatch( executed_shell_commands[1], f"*conda create -p {str(proj.path / '.tox' / 'py123')} python=* --yes --quiet*") def test_conda_create_with_name(tox_project, mock_conda_env_runner): @@ -41,16 +36,7 @@ def test_conda_create_with_name(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 - - create_env_cmd = executed_shell_commands[1] - - assert "conda" in create_env_cmd[0] - assert "create" == create_env_cmd[1] - assert "-n" == create_env_cmd[2] - assert "myenv" == create_env_cmd[3] - assert create_env_cmd[4].startswith("python=") - assert "--yes" == create_env_cmd[5] - assert "--quiet" == create_env_cmd[6] + assert fnmatch( executed_shell_commands[1], f"*conda create -n myenv python=* --yes --quiet*") def test_install_deps_no_conda(tox_project, mock_conda_env_runner): @@ -72,18 +58,11 @@ def test_install_deps_no_conda(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 3 - cmd = executed_shell_commands[2] - cmd_conda_prefix = " ".join(cmd[:5]) - cmd_pip_install = " ".join(cmd[5:]) - - assert cmd_conda_prefix.endswith( - f"conda run -p {str(proj.path / '.tox' / env_name)} --live-stream" - ) - - assert cmd_pip_install.startswith("python -I -m pip install") - assert "numpy" in cmd_pip_install - assert "astropy" in cmd_pip_install - assert "-r requirements.txt" in cmd_pip_install + pip_install_command = executed_shell_commands[2] + assert fnmatch(pip_install_command, f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream python -I -m pip install*") + assert "numpy" in pip_install_command + assert "astropy" in pip_install_command + assert "-r requirements.txt" in pip_install_command def test_install_conda_no_pip(tox_project, mock_conda_env_runner): @@ -103,19 +82,13 @@ def test_install_conda_no_pip(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 3 - cmd = executed_shell_commands[2] - cmd_conda_prefix = " ".join(cmd[:6]) - cmd_packages = " ".join(cmd[6:]) - - assert cmd_conda_prefix.endswith( - f"conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}" - ) - - assert "asdf" in cmd_packages - assert "pytest" in cmd_packages + conda_install_command = executed_shell_commands[2] + assert fnmatch(conda_install_command, f"*conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}*") + assert "asdf" in conda_install_command + assert "pytest" in conda_install_command # Make sure that python is explicitly given as part of every conda install # in order to avoid inadvertent upgrades of python itself. - assert "python=" in cmd_packages + assert "python=" in conda_install_command def test_install_conda_with_deps(tox_project, mock_conda_env_runner): @@ -138,8 +111,8 @@ def test_install_conda_with_deps(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 4 - cmd_conda = " ".join(executed_shell_commands[2]) - pip_cmd = " ".join(executed_shell_commands[3]) + cmd_conda = executed_shell_commands[2] + pip_cmd = executed_shell_commands[3] assert "conda install --quiet --yes -p" in cmd_conda assert "python -I -m pip install" in pip_cmd @@ -164,18 +137,12 @@ def test_conda_spec(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 3 - cmd = executed_shell_commands[2] - cmd_conda_prefix = " ".join(cmd[:6]) - cmd_packages = " ".join(cmd[6:]) - - assert cmd_conda_prefix.endswith( - f"conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}" - ) - - assert "astropy" in cmd_packages - assert "numpy" in cmd_packages - assert "python=" in cmd_packages - assert "--file=conda_spec.txt" in cmd_packages + conda_install_command = executed_shell_commands[2] + assert fnmatch(conda_install_command, f"*conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}*") + assert "astropy" in conda_install_command + assert "numpy" in conda_install_command + assert "python=" in conda_install_command + assert "--file=conda_spec.txt" in conda_install_command def test_conda_env(tmp_path, tox_project, mock_conda_env_runner): @@ -213,18 +180,7 @@ def open_mock_temp_file(*args, **kwargs): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 - - create_env_cmd = executed_shell_commands[1] - - assert "conda" in create_env_cmd[0] - assert "env" == create_env_cmd[1] - assert "create" == create_env_cmd[2] - assert "-p" == create_env_cmd[3] - assert str(proj.path / ".tox" / "py123") == create_env_cmd[4] - assert "--file" == create_env_cmd[5] - assert str(mock_temp_file) == create_env_cmd[6] - assert "--quiet" == create_env_cmd[7] - assert "--force" == create_env_cmd[8] + assert fnmatch(executed_shell_commands[1], f"*conda env create -p {str(proj.path / '.tox' / 'py123')} --file {str(mock_temp_file)} --quiet --force") # Check that the temporary file has the correct contents yaml = YAML() @@ -263,13 +219,8 @@ def test_conda_env_and_spec(tox_project, mock_conda_env_runner): create_env_cmd = executed_shell_commands[1] install_cmd = executed_shell_commands[2] - - assert "conda" in create_env_cmd[0] - assert "env" == create_env_cmd[1] - assert "create" == create_env_cmd[2] - - assert "conda" in install_cmd[0] - assert "install" == install_cmd[1] + assert fnmatch( create_env_cmd, f"*conda env create*") + assert fnmatch( install_cmd, f"*conda install*") assert "--file=conda_spec.txt" in install_cmd @@ -292,9 +243,7 @@ def test_conda_install_args(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 3 install_cmd = executed_shell_commands[2] - - assert "conda" in install_cmd[0] - assert "install" == install_cmd[1] + assert fnmatch( install_cmd, f"*conda install*") assert "--override-channels" in install_cmd @@ -315,7 +264,5 @@ def test_conda_create_args(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 2 create_cmd = executed_shell_commands[1] - - assert "conda" in create_cmd[0] - assert "create" == create_cmd[1] + assert fnmatch( create_cmd, f"*conda create*") assert "--override-channels" in create_cmd From b27855d1516d0af2a23994f6886d8c9e3969d3f0 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 18:07:03 +0100 Subject: [PATCH 30/39] Format. --- tests/test_cache.py | 16 ++++++++++------ tests/test_conda.py | 15 +++++++++------ tests/test_conda_env.py | 38 ++++++++++++++++++++++++++------------ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/tests/test_cache.py b/tests/test_cache.py index 93e362d..067eca1 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -2,11 +2,14 @@ from fnmatch import fnmatch + def assert_create_command(cmd): - assert fnmatch(cmd, "*conda create*") or fnmatch(cmd, "*conda env create*") + assert fnmatch(cmd, "*conda create*") or fnmatch(cmd, "*conda env create*") + def assert_install_command(cmd): - assert fnmatch(cmd, "*conda install*") + assert fnmatch(cmd, "*conda install*") + def test_conda_no_recreate(tox_project, mock_conda_env_runner): ini = """ @@ -39,6 +42,7 @@ def test_conda_no_recreate(tox_project, mock_conda_env_runner): assert_create_command(executed_shell_commands[1]) assert_install_command(executed_shell_commands[2]) + def test_conda_recreate_by_dependency_change(tox_project, mock_conda_env_runner): ini = """ [testenv:py123] @@ -91,7 +95,7 @@ def test_conda_recreate_by_env_file_path_change(tox_project, mock_conda_env_runn - pip: - pytest """ - + proj_1 = tox_project({"tox.ini": ini}) (proj_1.path / "conda-env-1.yml").write_text(yaml) outcome = proj_1.run("-e", "py123") @@ -136,7 +140,7 @@ def test_conda_recreate_by_env_file_content_change(tox_project, mock_conda_env_r - pip: - pytest """ - + proj = tox_project({"tox.ini": ini}) (proj.path / "conda-env.yml").write_text(yaml) outcome = proj.run("-e", "py123") @@ -176,7 +180,7 @@ def test_conda_recreate_by_spec_file_path_change(tox_project, mock_conda_env_run - pip: - pytest """ - + proj_1 = tox_project({"tox.ini": ini}) (proj_1.path / "conda_spec-1.txt").touch() outcome = proj_1.run("-e", "py123") @@ -210,7 +214,7 @@ def test_conda_recreate_by_spec_file_content_change(tox_project, mock_conda_env_ black numpy """ - + proj = tox_project({"tox.ini": ini}) (proj.path / "conda_spec.txt").write_text(txt) outcome = proj.run("-e", "py123") diff --git a/tests/test_conda.py b/tests/test_conda.py index a1f95fb..cbf1476 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -1,18 +1,21 @@ import os import shutil -import pytest +from fnmatch import fnmatch +import pytest import tox from tox.tox_env.errors import Fail import tox_conda.plugin -from fnmatch import fnmatch - def assert_conda_context(proj, env_name, shell_command, expected_command): - assert fnmatch(shell_command, f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream {expected_command}") - + assert fnmatch( + shell_command, + f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream {expected_command}", + ) + + def test_conda(cmd, initproj): # The path has a blank space on purpose for testing issue #119. initproj( @@ -55,7 +58,7 @@ def test_conda_run_command(tox_project, mock_conda_env_runner): outcome.assert_success() executed_shell_commands = mock_conda_env_runner - + assert len(executed_shell_commands) == 5 assert_conda_context(proj, "py123", executed_shell_commands[2], "python --version") assert_conda_context(proj, "py123", executed_shell_commands[3], "pytest") diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 319a88d..93c8693 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -1,12 +1,11 @@ """Conda environment creation and installation tests.""" import pathlib +from fnmatch import fnmatch from unittest.mock import patch from ruamel.yaml import YAML -from fnmatch import fnmatch - def test_conda_create(tox_project, mock_conda_env_runner): ini = """ @@ -20,7 +19,10 @@ def test_conda_create(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 - assert fnmatch( executed_shell_commands[1], f"*conda create -p {str(proj.path / '.tox' / 'py123')} python=* --yes --quiet*") + assert fnmatch( + executed_shell_commands[1], + f"*conda create -p {str(proj.path / '.tox' / 'py123')} python=* --yes --quiet*", + ) def test_conda_create_with_name(tox_project, mock_conda_env_runner): @@ -36,7 +38,7 @@ def test_conda_create_with_name(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 - assert fnmatch( executed_shell_commands[1], f"*conda create -n myenv python=* --yes --quiet*") + assert fnmatch(executed_shell_commands[1], f"*conda create -n myenv python=* --yes --quiet*") def test_install_deps_no_conda(tox_project, mock_conda_env_runner): @@ -59,7 +61,10 @@ def test_install_deps_no_conda(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 3 pip_install_command = executed_shell_commands[2] - assert fnmatch(pip_install_command, f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream python -I -m pip install*") + assert fnmatch( + pip_install_command, + f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream python -I -m pip install*", + ) assert "numpy" in pip_install_command assert "astropy" in pip_install_command assert "-r requirements.txt" in pip_install_command @@ -83,7 +88,10 @@ def test_install_conda_no_pip(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 3 conda_install_command = executed_shell_commands[2] - assert fnmatch(conda_install_command, f"*conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}*") + assert fnmatch( + conda_install_command, + f"*conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}*", + ) assert "asdf" in conda_install_command assert "pytest" in conda_install_command # Make sure that python is explicitly given as part of every conda install @@ -138,7 +146,10 @@ def test_conda_spec(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 3 conda_install_command = executed_shell_commands[2] - assert fnmatch(conda_install_command, f"*conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}*") + assert fnmatch( + conda_install_command, + f"*conda install --quiet --yes -p {str(proj.path / '.tox' / env_name)}*", + ) assert "astropy" in conda_install_command assert "numpy" in conda_install_command assert "python=" in conda_install_command @@ -180,7 +191,10 @@ def open_mock_temp_file(*args, **kwargs): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 - assert fnmatch(executed_shell_commands[1], f"*conda env create -p {str(proj.path / '.tox' / 'py123')} --file {str(mock_temp_file)} --quiet --force") + assert fnmatch( + executed_shell_commands[1], + f"*conda env create -p {str(proj.path / '.tox' / 'py123')} --file {str(mock_temp_file)} --quiet --force", + ) # Check that the temporary file has the correct contents yaml = YAML() @@ -219,8 +233,8 @@ def test_conda_env_and_spec(tox_project, mock_conda_env_runner): create_env_cmd = executed_shell_commands[1] install_cmd = executed_shell_commands[2] - assert fnmatch( create_env_cmd, f"*conda env create*") - assert fnmatch( install_cmd, f"*conda install*") + assert fnmatch(create_env_cmd, f"*conda env create*") + assert fnmatch(install_cmd, f"*conda install*") assert "--file=conda_spec.txt" in install_cmd @@ -243,7 +257,7 @@ def test_conda_install_args(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 3 install_cmd = executed_shell_commands[2] - assert fnmatch( install_cmd, f"*conda install*") + assert fnmatch(install_cmd, f"*conda install*") assert "--override-channels" in install_cmd @@ -264,5 +278,5 @@ def test_conda_create_args(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 2 create_cmd = executed_shell_commands[1] - assert fnmatch( create_cmd, f"*conda create*") + assert fnmatch(create_cmd, f"*conda create*") assert "--override-channels" in create_cmd From 9f761eb920d225221283cb14bf3a06581122c673 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 20:58:12 +0100 Subject: [PATCH 31/39] Add more tests. --- tests/test_conda.py | 82 +++++++++++++++++++++-------------------- tests/test_conda_env.py | 24 ++++++++++++ 2 files changed, 66 insertions(+), 40 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index cbf1476..6d96624 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -2,12 +2,6 @@ import shutil from fnmatch import fnmatch -import pytest -import tox -from tox.tox_env.errors import Fail - -import tox_conda.plugin - def assert_conda_context(proj, env_name, shell_command, expected_command): assert fnmatch( @@ -16,34 +10,6 @@ def assert_conda_context(proj, env_name, shell_command, expected_command): ) -def test_conda(cmd, initproj): - # The path has a blank space on purpose for testing issue #119. - initproj( - "pkg 1", - filedefs={ - "tox.ini": """ - [tox] - skipsdist=True - [testenv] - commands = python -c 'import sys, os; \ - print(os.path.exists(os.path.join(sys.prefix, "conda-meta")))' - """ - }, - ) - result = cmd("-v", "-e", "py") - result.assert_success() - - def index_of(m): - return next((i for i, l in enumerate(result.outlines) if l.startswith(m)), None) - - assert any( - "create --yes -p " in line - for line in result.outlines[index_of("py create: ") + 1 : index_of("py installed: ")] - ), result.output() - - assert result.outlines[-4] == "True" - - def test_conda_run_command(tox_project, mock_conda_env_runner): """Check that all the commands are run from an activated anaconda env.""" ini = """ @@ -65,11 +31,32 @@ def test_conda_run_command(tox_project, mock_conda_env_runner): assert_conda_context(proj, "py123", executed_shell_commands[4], "black") -def test_missing_conda(tox_project, mock_conda_env_runner, monkeypatch): +def test_missing_conda(tox_project, monkeypatch): """Check that an error is shown when the conda executable is not found.""" ini = """ - [tox] - require = tox-conda + [testenv:py123] + skip_install = True + runner = conda + """ + # Prevent conda from being found. + original_which = shutil.which + + def which(cmd, mode=os.F_OK | os.X_OK, path=None): + if cmd.endswith("conda"): + return None + return original_which(cmd, mode, path) + + monkeypatch.setattr(shutil, "which", which) + monkeypatch.delenv("_CONDA_EXE", raising=False) + monkeypatch.delenv("CONDA_EXE", raising=False) + + outcome = tox_project({"tox.ini": ini}).run("-e", "py123") + + outcome.assert_failed() + assert "Failed to find 'conda' executable." in outcome.out + +def test_missing_conda_fallback(tox_project, mock_conda_env_runner, monkeypatch): + ini = """ [testenv:py123] skip_install = True """ @@ -85,7 +72,22 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): monkeypatch.delenv("_CONDA_EXE", raising=False) monkeypatch.delenv("CONDA_EXE", raising=False) - with pytest.raises(Fail) as exc_info: - tox_project({"tox.ini": ini}).run("-e", "py123") + outcome = tox_project({"tox.ini": ini}).run("-e", "py123") + + outcome.assert_success() + executed_shell_commands = mock_conda_env_runner + # No conda commands should be run because virtualenv is used as a fallback. + assert len(executed_shell_commands) == 0 - assert str(exc_info.value) == "Failed to find 'conda' executable." +def test_conda_runner_overload(tox_project, mock_conda_env_runner, monkeypatch): + ini = """ + [testenv:py123] + skip_install = True + runner = virtualenv + """ + outcome = tox_project({"tox.ini": ini}).run("-e", "py123") + + outcome.assert_success() + executed_shell_commands = mock_conda_env_runner + # No conda commands should be run because virtualenv is used. + assert len(executed_shell_commands) == 0 \ No newline at end of file diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 93c8693..6d1b0ff 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -25,6 +25,30 @@ def test_conda_create(tox_project, mock_conda_env_runner): ) +def test_conda_create_with_package_install(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + """ + proj = tox_project({"tox.ini": ini}) + + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 3 + conda_create_command = executed_shell_commands[1] + pip_install_command = executed_shell_commands[2] + + assert fnmatch( + conda_create_command, + f"*conda create -p {str(proj.path / '.tox' / 'py123')} python=* --yes --quiet*", + ) + assert fnmatch( + pip_install_command, + f"*conda run -p {str(proj.path / '.tox' / 'py123')} --live-stream python -I -m pip install*", + ) + + def test_conda_create_with_name(tox_project, mock_conda_env_runner): ini = """ [testenv:py123] From ebe2c706262759b5f6b48fd45810725caa8f4c12 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 20:58:49 +0100 Subject: [PATCH 32/39] Format. --- tests/test_conda.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index 6d96624..4538fb2 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -55,6 +55,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): outcome.assert_failed() assert "Failed to find 'conda' executable." in outcome.out + def test_missing_conda_fallback(tox_project, mock_conda_env_runner, monkeypatch): ini = """ [testenv:py123] @@ -79,6 +80,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): # No conda commands should be run because virtualenv is used as a fallback. assert len(executed_shell_commands) == 0 + def test_conda_runner_overload(tox_project, mock_conda_env_runner, monkeypatch): ini = """ [testenv:py123] @@ -90,4 +92,4 @@ def test_conda_runner_overload(tox_project, mock_conda_env_runner, monkeypatch): outcome.assert_success() executed_shell_commands = mock_conda_env_runner # No conda commands should be run because virtualenv is used. - assert len(executed_shell_commands) == 0 \ No newline at end of file + assert len(executed_shell_commands) == 0 From fa0957914eaefc21d5011e73bae7ebb12e0818d1 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 21:00:11 +0100 Subject: [PATCH 33/39] Linting fixes. --- tests/conftest.py | 30 ++++-------------------------- tests/test_cache.py | 14 +------------- tests/test_conda_env.py | 25 +++++++++++++++++-------- tox_conda/plugin.py | 7 +++++-- 4 files changed, 27 insertions(+), 49 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 16757d1..2bd33a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,39 +1,17 @@ -import io -import os -import pathlib -import re -import subprocess from pathlib import Path -from types import ModuleType, TracebackType -from typing import Any, Callable, Dict, Optional, Sequence, Union -from unittest.mock import mock_open, patch +from types import TracebackType +from typing import Any, Dict, Optional, Sequence import pytest -import tox -import tox.run from pytest import MonkeyPatch from pytest_mock import MockerFixture -from ruamel.yaml import YAML -from tox.config.sets import EnvConfigSet -from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteStatus, Outcome -from tox.execute.request import ExecuteRequest, shell_cmd +from tox.execute.api import ExecuteInstance, ExecuteOptions, ExecuteStatus +from tox.execute.request import ExecuteRequest from tox.execute.stream import SyncWrite -from tox.plugin import manager from tox.pytest import CaptureFixture, ToxProject, ToxProjectCreator -from tox.report import LOGGER, OutErr -from tox.run import run as tox_run -from tox.run import setup_state as previous_setup_state -from tox.session.cmd.run.parallel import ENV_VAR_KEY -from tox.session.state import State -from tox.tox_env import api as tox_env_api -from tox.tox_env.api import ToxEnv from tox_conda.plugin import CondaEnvRunner -# from tox_conda.plugin import tox_testenv_create, tox_testenv_install_deps - -# pytest_plugins = "tox.pytest" - @pytest.fixture(name="tox_project") def init_fixture( diff --git a/tests/test_cache.py b/tests/test_cache.py index 067eca1..b228dc5 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -30,7 +30,7 @@ def test_conda_no_recreate(tox_project, mock_conda_env_runner): - pip: - pytest """ - for i in range(2): + for _ in range(2): proj = tox_project({"tox.ini": ini}) (proj.path / "conda-env.yml").write_text(yaml) outcome = proj.run("-e", "py123") @@ -169,18 +169,6 @@ def test_conda_recreate_by_spec_file_path_change(tox_project, mock_conda_env_run skip_install = True conda_spec = conda_spec-2.txt """ - yaml = """ - name: tox-conda - channels: - - conda-forge - - nodefaults - dependencies: - - numpy - - astropy - - pip: - - pytest - """ - proj_1 = tox_project({"tox.ini": ini}) (proj_1.path / "conda_spec-1.txt").touch() outcome = proj_1.run("-e", "py123") diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index 6d1b0ff..c019fca 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -45,7 +45,10 @@ def test_conda_create_with_package_install(tox_project, mock_conda_env_runner): ) assert fnmatch( pip_install_command, - f"*conda run -p {str(proj.path / '.tox' / 'py123')} --live-stream python -I -m pip install*", + ( + f"*conda run -p {str(proj.path / '.tox' / 'py123')} --live-stream" + " python -I -m pip install*" + ), ) @@ -62,7 +65,7 @@ def test_conda_create_with_name(tox_project, mock_conda_env_runner): executed_shell_commands = mock_conda_env_runner assert len(executed_shell_commands) == 2 - assert fnmatch(executed_shell_commands[1], f"*conda create -n myenv python=* --yes --quiet*") + assert fnmatch(executed_shell_commands[1], "*conda create -n myenv python=* --yes --quiet*") def test_install_deps_no_conda(tox_project, mock_conda_env_runner): @@ -87,7 +90,10 @@ def test_install_deps_no_conda(tox_project, mock_conda_env_runner): pip_install_command = executed_shell_commands[2] assert fnmatch( pip_install_command, - f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream python -I -m pip install*", + ( + f"*conda run -p {str(proj.path / '.tox' / env_name)} --live-stream" + " python -I -m pip install*" + ), ) assert "numpy" in pip_install_command assert "astropy" in pip_install_command @@ -217,7 +223,10 @@ def open_mock_temp_file(*args, **kwargs): assert len(executed_shell_commands) == 2 assert fnmatch( executed_shell_commands[1], - f"*conda env create -p {str(proj.path / '.tox' / 'py123')} --file {str(mock_temp_file)} --quiet --force", + ( + f"*conda env create -p {str(proj.path / '.tox' / 'py123')} " + f" --file {str(mock_temp_file)} --quiet --force" + ), ) # Check that the temporary file has the correct contents @@ -257,8 +266,8 @@ def test_conda_env_and_spec(tox_project, mock_conda_env_runner): create_env_cmd = executed_shell_commands[1] install_cmd = executed_shell_commands[2] - assert fnmatch(create_env_cmd, f"*conda env create*") - assert fnmatch(install_cmd, f"*conda install*") + assert fnmatch(create_env_cmd, "*conda env create*") + assert fnmatch(install_cmd, "*conda install*") assert "--file=conda_spec.txt" in install_cmd @@ -281,7 +290,7 @@ def test_conda_install_args(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 3 install_cmd = executed_shell_commands[2] - assert fnmatch(install_cmd, f"*conda install*") + assert fnmatch(install_cmd, "*conda install*") assert "--override-channels" in install_cmd @@ -302,5 +311,5 @@ def test_conda_create_args(tox_project, mock_conda_env_runner): assert len(executed_shell_commands) == 2 create_cmd = executed_shell_commands[1] - assert fnmatch(create_cmd, f"*conda create*") + assert fnmatch(create_cmd, "*conda create*") assert "--override-channels" in create_cmd diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 2d9b9e8..14a9db7 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -11,7 +11,7 @@ from io import BytesIO, TextIOWrapper from pathlib import Path from time import sleep -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional from ruamel.yaml import YAML from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteRequest, SyncWrite @@ -211,7 +211,10 @@ def _generate_env_create_command( yaml.dump(env_file, tmp_env_file) tmp_env_file.close() - cmd = f"'{conda_exe}' env create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}' --file '{tmp_env_file.name}' --quiet --force" + cmd = ( + f"'{conda_exe}' env create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}'" + f" --file '{tmp_env_file.name}' --quiet --force" + ) def tear_down(): return Path(tmp_env_file.name).unlink() From 63faefc6126b3117bd7b56dab4e15d688aae6782 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 2 Jun 2023 21:21:57 +0100 Subject: [PATCH 34/39] Run tests with pytest directly. --- requirements_tests.txt | 8 ++++++++ tests/conftest.py | 2 ++ tests/test_conda.py | 5 +++++ tests/test_conda_env.py | 2 +- tox.ini | 6 +++++- 5 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 requirements_tests.txt diff --git a/requirements_tests.txt b/requirements_tests.txt new file mode 100644 index 0000000..35e25bf --- /dev/null +++ b/requirements_tests.txt @@ -0,0 +1,8 @@ +-e . +devpi_process +pytest +pytest-cov +pytest-mock +pytest-ordering +pytest-timeout +tox>=4,<5 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 2bd33a6..9701ec4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,5 +95,7 @@ def mock_execute_instance_factory( return original_execute_instance_factor(request, options, out, err) monkeypatch.setattr(CondaEnvRunner, "_execute_instance_factory", mock_execute_instance_factory) + monkeypatch.setenv("CONDA_EXE", "conda") + monkeypatch.setenv("CONDA_DEFAULT_ENV", "test-env") yield shell_cmds diff --git a/tests/test_conda.py b/tests/test_conda.py index 4538fb2..dda5384 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -2,6 +2,8 @@ import shutil from fnmatch import fnmatch +import pytest + def assert_conda_context(proj, env_name, shell_command, expected_command): assert fnmatch( @@ -56,6 +58,8 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): assert "Failed to find 'conda' executable." in outcome.out +# This test must run first to avoid collisions with other tests. +@pytest.mark.first def test_missing_conda_fallback(tox_project, mock_conda_env_runner, monkeypatch): ini = """ [testenv:py123] @@ -72,6 +76,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): monkeypatch.setattr(shutil, "which", which) monkeypatch.delenv("_CONDA_EXE", raising=False) monkeypatch.delenv("CONDA_EXE", raising=False) + monkeypatch.delenv("CONDA_DEFAULT_ENV", raising=False) outcome = tox_project({"tox.ini": ini}).run("-e", "py123") diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index c019fca..c6dca0b 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -224,7 +224,7 @@ def open_mock_temp_file(*args, **kwargs): assert fnmatch( executed_shell_commands[1], ( - f"*conda env create -p {str(proj.path / '.tox' / 'py123')} " + f"*conda env create -p {str(proj.path / '.tox' / 'py123')}" f" --file {str(mock_temp_file)} --quiet --force" ), ) diff --git a/tox.ini b/tox.ini index b445310..a968501 100644 --- a/tox.ini +++ b/tox.ini @@ -20,8 +20,12 @@ setenv = PYTHONDONTWRITEBYTECODE = 1 VIRTUALENV_DOWNLOAD = 0 deps = + devpi_process + pytest + pytest-cov + pytest-mock pytest-timeout - tox[testing]>=3.8.1,<4 + tox>=4,<5 commands = pytest {posargs: \ --junitxml {toxworkdir}/junit.{envname}.xml --cov {envsitepackagesdir}/tox_conda --cov tests \ From bc0cf59e62fe08be3026ac9363b437f1f46f2ed9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jun 2023 23:11:42 +0000 Subject: [PATCH 35/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements_tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_tests.txt b/requirements_tests.txt index 35e25bf..ebb84bf 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -5,4 +5,4 @@ pytest-cov pytest-mock pytest-ordering pytest-timeout -tox>=4,<5 \ No newline at end of file +tox>=4,<5 From 7935e756a4cfc08f4d1af6e3ec21def78ae73ac6 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 19 Jan 2024 17:33:19 +0000 Subject: [PATCH 36/39] Add conda_python option. --- tests/test_conda_env.py | 34 +++++++++++++++++++ tox_conda/plugin.py | 74 ++++++++++++++++++++++++++++++----------- 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/tests/test_conda_env.py b/tests/test_conda_env.py index c6dca0b..611a733 100644 --- a/tests/test_conda_env.py +++ b/tests/test_conda_env.py @@ -68,6 +68,40 @@ def test_conda_create_with_name(tox_project, mock_conda_env_runner): assert fnmatch(executed_shell_commands[1], "*conda create -n myenv python=* --yes --quiet*") +def test_conda_create_with_conda_python(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_name = myenv + conda_python = python3.10 + """ + proj = tox_project({"tox.ini": ini}) + + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 2 + assert fnmatch(executed_shell_commands[1], "*conda create -n myenv python=3.10 --yes --quiet*") + + +def test_conda_create_with_conda_python_as_pypy(tox_project, mock_conda_env_runner): + ini = """ + [testenv:py123] + skip_install = True + conda_name = myenv + conda_python = pypy3.9 + """ + proj = tox_project({"tox.ini": ini}) + + outcome = proj.run("-e", "py123") + outcome.assert_success() + + executed_shell_commands = mock_conda_env_runner + assert len(executed_shell_commands) == 2 + assert fnmatch(executed_shell_commands[1], "*conda create -n myenv pypy3.9 pip --yes --quiet*") + + def test_install_deps_no_conda(tox_project, mock_conda_env_runner): env_name = "py123" ini = f""" diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 14a9db7..47f1828 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -98,19 +98,40 @@ def _package_tox_env_type(self) -> str: def _external_pkg_tox_env_type(self) -> str: return "virtualenv-cmd-builder" - def _get_python_env_version(self): + def _get_python_packages(self): # Try to use base_python config - match = re.match(r"python(\d)(?:\.(\d+))?(?:\.?(\d))?", self.conf["base_python"][0]) - if match: - groups = match.groups() - version = groups[0] - if groups[1]: - version += ".{}".format(groups[1]) - if groups[2]: - version += ".{}".format(groups[2]) - return version + conda_python = self.conf["conda_python"] + if not conda_python: + # If not specified, use the base_python version + return [f"python={self.base_python.version_dot}"] + + match = re.match(r"(python|pypy)(\d)(?:\.(\d+))?(?:\.?(\d))?", conda_python) + if not match: + raise Fail( + f"Invalid conda_python value '{conda_python}'. " + "Must be in the form of 'pythonX.Y' or 'pypyX.Y'." + ) + + groups = match.groups() + interpreter = groups[0] + + version = "" + if groups[1]: + version = groups[1] + if groups[2]: + version += ".{}".format(groups[2]) + if groups[3]: + version += ".{}".format(groups[3]) + + if interpreter == "pypy": + # PyPy doesn't pull pip as a dependency, so we need to manually specify it + packages = [f"pypy{version}", "pip"] + elif interpreter == "python": + packages = [f"python={version}"] else: - return self.base_python.version_dot + raise Fail(f"Unknown interpreter {interpreter}") + + return packages @property def runs_on_platform(self) -> str: @@ -165,17 +186,18 @@ def python_cache(self) -> Dict[str, Any]: def create_python_env(self) -> None: conda_exe = find_conda() - python_version = self._get_python_env_version() - python = f"python={python_version}" + python_packages = self._get_python_packages() + python_packages = " ".join(python_packages) + conda_cache_conf = self.python_cache()["conda"] if self.conf["conda_env"]: create_command, tear_down = CondaEnvRunner._generate_env_create_command( - conda_exe, python, conda_cache_conf + conda_exe, python_packages, conda_cache_conf ) else: create_command, tear_down = CondaEnvRunner._generate_create_command( - conda_exe, python, conda_cache_conf + conda_exe, python_packages, conda_cache_conf ) try: create_command_args = shlex.split(create_command) @@ -184,7 +206,7 @@ def create_python_env(self) -> None: tear_down() install_command = CondaEnvRunner._generate_install_command( - conda_exe, python, conda_cache_conf + conda_exe, python_packages, conda_cache_conf ) if install_command: install_command_args = shlex.split(install_command) @@ -222,10 +244,12 @@ def tear_down(): return cmd, tear_down @staticmethod - def _generate_create_command(conda_exe: Path, python: str, conda_cache_conf: Dict[str, str]): + def _generate_create_command( + conda_exe: Path, python_packages: str, conda_cache_conf: Dict[str, str] + ): cmd = ( f"'{conda_exe}' create {conda_cache_conf['env_spec']} '{conda_cache_conf['env']}'" - f" {python} --yes --quiet" + f" {python_packages} --yes --quiet" ) for arg in conda_cache_conf.get("create_args", []): cmd += f" '{arg}'" @@ -236,7 +260,9 @@ def tear_down(): return cmd, tear_down @staticmethod - def _generate_install_command(conda_exe: Path, python: str, conda_cache_conf: Dict[str, str]): + def _generate_install_command( + conda_exe: Path, python_packages: str, conda_cache_conf: Dict[str, str] + ): # Check if there is anything to install if "deps" not in conda_cache_conf and "spec" not in conda_cache_conf: return None @@ -257,7 +283,7 @@ def _generate_install_command(conda_exe: Path, python: str, conda_cache_conf: Di # python in this environment. If any of the requirements are in conflict # with the installed python version, installation will fail (which is what # we want). - cmd += f" {python}" + cmd += f" {python_packages}" for dep in conda_cache_conf.get("deps", []): cmd += f" {dep}" @@ -450,6 +476,14 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: default=None, ) + env_conf.add_config( + "conda_python", + of_type=str, + desc="Specifies the name of the Python interpreter (python or pypy) and its version in the conda " + 'environment. By default, it uses the "python" interpreter and the currently active version.', + default=None, + ) + env_conf.add_config( "conda_env", of_type=str, From 305ff7f135ed49537228bebbf05b35d1c5956409 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 19 Jan 2024 17:45:58 +0000 Subject: [PATCH 37/39] Lint and type fixes. --- tox_conda/plugin.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 47f1828..5b1daa2 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -11,14 +11,19 @@ from io import BytesIO, TextIOWrapper from pathlib import Path from time import sleep -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING from ruamel.yaml import YAML -from tox.execute.api import Execute, ExecuteInstance, ExecuteOptions, ExecuteRequest, SyncWrite +from tox.execute.api import ( + Execute, + ExecuteInstance, + ExecuteOptions, + ExecuteRequest, + StdinSource, + SyncWrite, +) from tox.execute.local_sub_process import LocalSubProcessExecuteInstance, LocalSubProcessExecutor from tox.plugin import impl -from tox.plugin.spec import EnvConfigSet, State, ToxEnvRegister -from tox.tox_env.api import StdinSource, ToxEnvCreateArgs from tox.tox_env.errors import Fail from tox.tox_env.installer import Installer from tox.tox_env.python.api import PythonInfo, VersionInfo @@ -26,6 +31,10 @@ from tox.tox_env.python.pip.req_file import PythonDeps from tox.tox_env.python.runner import PythonRun +if TYPE_CHECKING: + from tox.plugin.spec import EnvConfigSet, State, ToxEnvRegister + from tox.tox_env.api import ToxEnvCreateArgs + __all__ = [] @@ -39,7 +48,7 @@ def _default_execute_instance_factory( _execute_instance_factory: Callable = _default_execute_instance_factory - def __init__(self, create_args: ToxEnvCreateArgs) -> None: + def __init__(self, create_args: "ToxEnvCreateArgs") -> None: self._installer = None self._executor = None self._external_executor = None @@ -456,7 +465,7 @@ def _ensure_python_env_exists(self) -> None: @impl -def tox_register_tox_env(register: ToxEnvRegister) -> None: # noqa: U100 +def tox_register_tox_env(register: "ToxEnvRegister") -> None: # noqa: U100 register.add_run_env(CondaEnvRunner) try: # Change the defaukt runner only if conda is available @@ -468,7 +477,7 @@ def tox_register_tox_env(register: ToxEnvRegister) -> None: # noqa: U100 @impl -def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: +def tox_add_env_config(env_conf: "EnvConfigSet", state: "State") -> None: env_conf.add_config( "conda_name", of_type=str, @@ -479,8 +488,9 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: env_conf.add_config( "conda_python", of_type=str, - desc="Specifies the name of the Python interpreter (python or pypy) and its version in the conda " - 'environment. By default, it uses the "python" interpreter and the currently active version.', + desc="Specifies the name of the Python interpreter (python or pypy) and its version " + "in the conda environment. By default, it uses the 'python' interpreter and the " + "currently active version.", default=None, ) From 023cc5444be4a9dd1da9c3c38d1b982566fc2b9c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jan 2024 17:46:17 +0000 Subject: [PATCH 38/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tox_conda/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox_conda/plugin.py b/tox_conda/plugin.py index 5b1daa2..2c22171 100644 --- a/tox_conda/plugin.py +++ b/tox_conda/plugin.py @@ -11,7 +11,7 @@ from io import BytesIO, TextIOWrapper from pathlib import Path from time import sleep -from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional from ruamel.yaml import YAML from tox.execute.api import ( From 97f68b8f9879ebdf81323beede4b9ad02f7d3748 Mon Sep 17 00:00:00 2001 From: Tibor Takacs Date: Fri, 19 Jan 2024 17:59:40 +0000 Subject: [PATCH 39/39] Try fixing tests. --- .github/workflows/check.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c8fa45c..9923fbd 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -59,7 +59,10 @@ jobs: - name: setup test suite run: tox -vv --notest - name: run test suite - run: tox --skip-pkg-install + run: | + . .tox/py310/bin/activate + cd tests + pytest . --timeout 180 --durations 5 env: PYTEST_ADDOPTS: "-vv --durations=20" CI_RUN: "yes"