From 6a68ef77579228da61b45610b8d8945efeca0c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Ftay=20Fabry?= Date: Tue, 21 May 2024 20:18:53 +0200 Subject: [PATCH] add url handling (#24) * support custom dependencies * add tests * fmt * add generate_environment function and allow additional list of requirements * simplify and add tests * simplify * refactor * update create_from_definition * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update generate_environment.py * Update generate_environment.py * use package name in pip requirements again a relative local directory has du be indicated like file:/../directory * update tests and local pip handling * simplify * add docstrings * fix * fix pip formatting * ad definition test * update readme * add missing file --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 15 ++ .gitignore | 167 +++++++++++++++++++ README.md | 69 +++++++- action.yml | 15 +- pydeps2env/__init__.py | 12 +- pydeps2env/environment.py | 188 +++++++++++++++------- pydeps2env/generate_environment.py | 139 ++++++++++++++-- pyproject.toml | 6 + test/definition.yaml | 29 ++++ test/local.yaml | 8 + test/pyproject.toml | 1 + test/requirements.txt | 1 + test/test_environment.py | 61 +++++++ test/test_package/pyproject.toml | 31 ++++ test/test_package/testproject/__init__.py | 7 + 15 files changed, 666 insertions(+), 83 deletions(-) create mode 100644 .gitignore create mode 100644 test/definition.yaml create mode 100644 test/local.yaml create mode 100644 test/test_environment.py create mode 100644 test/test_package/pyproject.toml create mode 100644 test/test_package/testproject/__init__.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6eec1da..b258b1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,21 @@ on: [push, pull_request, workflow_dispatch] jobs: + pytest: + name: pytest + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: pip install package + run: pip install .[test] + - name: run pytest + run: pytest + demo_job: name: action runs-on: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e0ace01 --- /dev/null +++ b/.gitignore @@ -0,0 +1,167 @@ +# custom folders +demo/ + +pydeps2env/_version.py + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ diff --git a/README.md b/README.md index 8cd04e3..b6cc488 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # pydeps2env An easy way to create conda environment files from you python project dependencies. -Creates a conda `environment.yml` file from python package dependencies listed in a `pyproject.toml` or `setup.cfg` file. +Creates a conda `environment.yml` file from python package dependencies listed in on or multiple different source formats like `pyproject.toml`, `setup.cfg`, `environment.yaml` or `requirements.txt` files. The project contains - GitHub action @@ -42,7 +42,7 @@ test = ["pytest"] pip_only = ["bidict"] ``` -The default parameters will output this sorted `environment.yml` (note that the `python` dependency will always be the first item on the list): +The default parameters will output this sorted `environment.yml` (note that the `python` dependency specification will always be the first item on the list): ```yaml channels: @@ -74,13 +74,71 @@ dependencies: - bidict ``` -## configuration options +## basic usage (python) + +Create an `Environment` using Python and export it to an `environment.yaml` file. + +```python +from pydeps2env import Environment + +env = Environment("./test/pyproject.toml[doc]") +env.export("my_environment.yaml") +``` + +## basic usage (command line) + +Combine multiple source files into a single environment file (including build dependencies). +Install pandas using `pip`. + +```bash +pydeps2env ./test/setup.cfg ./test/pyproject.toml[doc] ./test/environment.yaml ./test/requirements.txt -o output.yaml -c defaults --extras test -b include --pip pandas +``` + +## advanced usage (definition file) + +Users can store complex configurations in a single yaml file and create the desired output using `create_from_definition`. +Example definition file: + +```yaml +# the target file to create +output: test-configuration.yaml +# default name of the environment +name: test-env +# conda channels to include +channels: +- conda-forge +- defaults +# list of source files that define sub environments +# these will be loaded as Environment() +sources: +- ./test/environment.yaml +- ./test/local.yaml +- ./test/pyproject.toml[doc] +- ./test/requirements.txt +- https://github.com/CagtayFabry/pydeps2env/blob/custom_deps/pyproject.toml +# extras to apply to all sources and packages +extras: +- test +# dependencies that should be removed after collection +remove: +- pyyaml +additional_requirements: +- urllib3 +# include build system dependencies +# list of dependencies that must be pip installed (excluding auto-sorted depedencies like urls) +pip: +- urllib3 +include_build_system: include + +``` + +## configuration options (GitHub action) To customize the output the input options are available to the action: ### files -Specify the location of the `'setup.cfg'` or `'pyproject.toml'` files to parse. (defaults to `'pyproject.toml'`) +Specify the location of the dependencies files to parse. (defaults to 'pyproject.toml') Multiple files can be listed. This will result in a combined environment file. ### output: @@ -105,7 +163,8 @@ is `'omit'` so no setup dependencies will be installed). ### pip List of packages to install via `pip` instead of `conda`. -The dependencies will be listet under the `pip:` section in the environment file. +The dependencies will be listed under the `pip:` section in the environment file. +If a dependency is listed with its URN, it will always be installed via pip (e.g. `pydeps2env @ git+https://github.com/CagtayFabry/pydeps2env`) ## example diff --git a/action.yml b/action.yml index ac0d34c..2d21d56 100644 --- a/action.yml +++ b/action.yml @@ -7,6 +7,7 @@ inputs: files: description: >- Specify the location of the dependencies files to parse. (defaults to 'pyproject.toml') + Separate a list of multiple source files by spaces (e.g. 'pyproject.toml[test] requirements.txt'). required: true default: pyproject.toml output: @@ -16,25 +17,27 @@ inputs: default: environment.yml channels: description: >- - List the conda channels to include in the environment file. (defaults to 'defaults') + List the conda channels to include in the environment file. (defaults to 'conda-forge') Separate a list of multiple channels by spaces (e.g. 'conda-forge defaults'). required: true - default: defaults + default: conda-forge extras: description: >- - Specify one or more optional [extras_require] sections to add to the environment - (e.g. 'test' to include package that you would normally install with 'pip install pkg[test]') + Specify one or more optional [extras_require] sections to add to all source files (s.a.). + (e.g. 'test' to include package that you would normally install with 'pip install pkg[test]'). + Note that for individual sources in 'files', the '[extra]' syntax is also possible. required: false build_system: description: >- - if set to 'include' the dependencies listed under [build-system] or [options]:setup_requires will be added to the environment - (default is 'omit' so no setup dependencies will be installed) + if set to 'include' the dependencies listed under [build-system] (for 'pyproject.toml') or [options]:setup_requires (for 'setup.cfg') will be added to the environment + (default is 'omit' so no build system dependencies will be installed) required: false default: omit pip: description: >- List of packages to install via pip instead of conda. The dependencies will be listet under the pip section in the environment file. + If a dependency is listed with its URN, it will always be installed via pip (e.g. 'pydeps2env @ git+https://github.com/CagtayFabry/pydeps2env') required: false runs: using: composite diff --git a/pydeps2env/__init__.py b/pydeps2env/__init__.py index c3252df..ecaa80b 100644 --- a/pydeps2env/__init__.py +++ b/pydeps2env/__init__.py @@ -1,8 +1,18 @@ """pydeps2env: helps to generate conda environment files from python package dependencies.""" from .environment import Environment +from .generate_environment import ( + create_environment, + create_environment_file, + create_from_definition, +) -__all__ = ["Environment"] +__all__ = [ + "Environment", + "create_environment", + "create_environment_file", + "create_from_definition", +] try: from ._version import __version__ diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py index 81f389c..c6d7f56 100644 --- a/pydeps2env/environment.py +++ b/pydeps2env/environment.py @@ -1,13 +1,14 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, InitVar from packaging.requirements import Requirement from pathlib import Path from collections import defaultdict import configparser import sys import yaml -import warnings +from io import StringIO, BytesIO +from warnings import warn if sys.version_info < (3, 11): import tomli as tomllib @@ -26,7 +27,7 @@ def split_extras(filename: str) -> tuple[str, set]: filename, extras = filename.split("[", 1) extras = set(extras.split("]", 1)[0].split(",")) else: - extras = {} + extras = set() return filename, extras @@ -44,6 +45,14 @@ def add_requirement( requirements[req.name] = req elif mode == "combine": requirements[req.name].specifier &= req.specifier + if req.url: + if requirements[req.name].url and requirements[req.name].url != req.url: + warn( + f"Replacing url for package {req.name}: {requirements[req.name].url} -> req.url", + UserWarning, + stacklevel=1, + ) + requirements[req.name].url = req.url elif mode == "replace": requirements[req.name] = req else: @@ -66,81 +75,114 @@ class Environment: filename: str | Path channels: list[str] = field(default_factory=lambda: ["conda-forge"]) extras: set[str] | list[str] = field(default_factory=set) - pip_packages: set[str] = field(default_factory=set) # install via pip + pip_packages: set[str] = field( + default_factory=set + ) # names of packages to install via pip + extra_requirements: InitVar[list[str]] = None requirements: dict[str, Requirement] = field(default_factory=dict, init=False) build_system: dict[str, Requirement] = field(default_factory=dict, init=False) - def __post_init__(self): + def __post_init__(self, extra_requirements): + print(self.filename) # cleanup duplicates etc. self.extras = set(self.extras) self.channels = list(dict.fromkeys(self.channels)) self.pip_packages = set(self.pip_packages) + if self.filename: + self._read_source() + + if extra_requirements: + self.add_requirements(extra_requirements) + + # packages with url specification must be pip installed + self.pip_packages |= {req.name for req in self.requirements.values() if req.url} + + def add_requirements(self, requirements: list[str]): + """Add a list of additional requirements to the environment.""" + + for req in requirements: + add_requirement(req, self.requirements) + + def add_build_system(self, requirements: list[str]): + """Manually add a list of additional requirements to the build system specification.""" + + for req in requirements: + add_requirement(req, self.build_system) + + def _read_source(self): + """Read and parse source definition and add requirements.""" + + # parse and remove extra specs from filename if isinstance(self.filename, str): self.filename, extras = split_extras(self.filename) self.extras |= set(extras) - if Path(self.filename).suffix == ".toml": - self.load_pyproject() - elif Path(self.filename).suffix == ".cfg": - self.load_config() - elif Path(self.filename).suffix in [".yaml", ".yml"]: - self.load_yaml() - elif Path(self.filename).suffix in [".txt"]: - self.load_txt() + # store suffix for later parsing + self._suffix = Path(self.filename).suffix + + # read file contents into bytes + _filename = self.filename + if isinstance(_filename, str) and _filename.startswith("http"): + import urllib.request + + if "/github.com/" in _filename: + _filename = _filename.replace( + "/github.com/", "/raw.githubusercontent.com/" + ) + _filename = _filename.replace("/blob/", "/") + with urllib.request.urlopen(_filename) as f: + _contents: bytes = f.read() # read raw content into bytes + else: + with open(_filename, "rb") as f: + _contents: bytes = f.read() + + if self._suffix == ".toml": + self.load_pyproject(_contents) + elif self._suffix == ".cfg": + self.load_config(_contents) + elif self._suffix in [".yaml", ".yml"]: + self.load_yaml(_contents) + elif self._suffix in [".txt"]: + self.load_txt(_contents) else: raise ValueError(f"Unsupported input {self.filename}") - def load_pyproject(self): + def load_pyproject(self, contents: bytes): """Load contents from a toml file (assume pyproject.toml layout).""" - with open(self.filename, "rb") as fh: - cp = defaultdict(dict, tomllib.load(fh)) - - if python := cp["project"].get("requires-python"): - add_requirement("python" + python, self.requirements) - for dep in cp.get("project").get("dependencies"): - add_requirement(dep, self.requirements) + tomldict = tomllib.load(BytesIO(contents)) + cp = defaultdict(dict, tomldict) - for dep in cp.get("build-system").get("requires"): - add_requirement(dep, self.build_system) + if python := cp["project"].get("requires-python"): + self.add_requirements(["python" + python]) + self.add_requirements(cp.get("project").get("dependencies")) + self.add_build_system(cp.get("build-system").get("requires")) for e in self.extras: - extra_deps = cp.get("project").get("optional-dependencies").get(e) - if not extra_deps: - continue - for dep in extra_deps: - add_requirement(dep, self.requirements) + self.add_requirements(cp.get("project").get("optional-dependencies").get(e)) - def load_config(self): + def load_config(self, contents: bytes): """Load contents from a cfg file (assume setup.cfg layout).""" cp = configparser.ConfigParser( converters={ "list": lambda x: [i.strip() for i in x.split("\n") if i.strip()] } ) - cp.read(self.filename) - if python := cp.get("options", "python_requires"): - add_requirement("python" + python, self.requirements) + cp.read_string(contents.decode("UTF-8")) - for dep in cp.getlist("options", "install_requires"): - add_requirement(dep, self.requirements) - - for dep in cp.getlist("options", "setup_requires"): - add_requirement(dep, self.build_system) + if python := cp.get("options", "python_requires"): + self.add_requirements(["python" + python]) + self.add_requirements(cp.getlist("options", "install_requires")) + self.add_build_system(cp.getlist("options", "setup_requires")) for e in self.extras: - extra_deps = cp.getlist("options.extras_require", e) - if not extra_deps: - continue - for dep in extra_deps: - add_requirement(dep, self.requirements) + self.add_requirements(cp.getlist("options.extras_require", e)) - def load_yaml(self): + def load_yaml(self, contents: bytes): """Load a conda-style environment.yaml file.""" - with open(self.filename, "r") as f: - env = yaml.load(f.read(), yaml.SafeLoader) + env = yaml.load(contents.decode(), yaml.SafeLoader) self.channels += env.get("channels", []) self.channels = list(dict.fromkeys(self.channels)) @@ -155,31 +197,49 @@ def load_yaml(self): self.pip_packages |= {req.name} add_requirement(req, self.requirements) - def load_txt(self): + def load_txt(self, contents: bytes): """Load simple list of requirements from txt file.""" - with open(self.filename, "r") as f: - deps = f.readlines() + deps = StringIO(contents.decode()).readlines() - for dep in deps: - add_requirement(dep, self.requirements) + self.add_requirements([dep.strip() for dep in deps]) def _get_dependencies( - self, include_build_system: bool = True + self, + include_build_system: bool = True, + remove: list[str] = None, ) -> tuple[list[str], list[str]]: """Get the default conda environment entries.""" + if remove is None: + remove = [] + reqs = self.requirements.copy() if include_build_system: reqs = combine_requirements(reqs, self.build_system) _python = reqs.pop("python", None) - deps = [str(r) for r in reqs.values() if r.name not in self.pip_packages] + _pip_packages = self.pip_packages + # _pip_packages |= {r.name for r in reqs.values() if r.url} + + deps = [ + str(r) + for r in reqs.values() + if not r.url # install via pip + and r.name not in _pip_packages + and r.name not in remove + ] deps.sort(key=str.lower) if _python: deps = [str(_python)] + deps - pip = [str(r) for r in reqs.values() if r.name in self.pip_packages] + pip = [ + r + for r in reqs.values() + if (r.name in _pip_packages or r.url) and r.name not in remove + ] + # string formatting + pip = [str(r) if not r.url else f"{r.name}@ {r.url}" for r in pip] pip.sort(key=str.lower) return deps, pip @@ -188,15 +248,29 @@ def export( self, outfile: str | Path = "environment.yaml", include_build_system: bool = True, + remove: list[str] = None, + name: str = None, ): - deps, pip = self._get_dependencies(include_build_system=include_build_system) + """Export the environment to a yaml or txt file.""" + if remove is None: + remove = [] - conda_env = {"channels": self.channels, "dependencies": deps.copy()} + deps, pip = self._get_dependencies( + include_build_system=include_build_system, remove=remove + ) + + conda_env = { + "name": name, + "channels": self.channels, + "dependencies": deps.copy(), + } if pip: if "pip" not in self.requirements: conda_env["dependencies"] += ["pip"] conda_env["dependencies"] += [{"pip": pip}] + conda_env = {k: v for k, v in conda_env.items() if v} + if outfile is None: return conda_env @@ -208,11 +282,11 @@ def export( outfile.writelines("\n".join(deps)) else: if p.suffix not in [".yaml", ".yml"]: - warnings.warn( + warn( f"Unknown environment format `{p.suffix}`, generating conda yaml output." ) with open(p, "w") as outfile: - yaml.dump(conda_env, outfile, default_flow_style=False) + yaml.dump(conda_env, outfile, default_flow_style=False, sort_keys=False) def combine(self, other: Environment): """Merge other Environment requirements into this Environment.""" diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py index 07a6154..2c97ed9 100644 --- a/pydeps2env/generate_environment.py +++ b/pydeps2env/generate_environment.py @@ -2,6 +2,8 @@ from pathlib import Path +import yaml + try: from pydeps2env.environment import Environment, split_extras except ModuleNotFoundError: # try local file if not installed @@ -9,20 +11,119 @@ def create_environment_file( - filename: list[str], - output_file: str, - channels: list[str], - extras: list[str], - pip: set[str], - include_build_system: bool = False, + sources: list[str], + output: str = "environment.yml", + *, + channels: list[str] = None, + extras: list[str] = None, + pip: set[str] = None, + additional_requirements: list[str] = None, + remove: set[str] = None, + include_build_system: str = "omit", + name: str = None, ): + """Create an environment file from multiple source files and additional requirements. + + Parameters + ---------- + sources + The list of source files to combine. + output + The output filename to generate + channels + Conda channels to include. + extras + Extras specification to apply to all sources. + pip + List of dependencies to install via pip. + additional_requirements + Additional requirements to include in the environment. + remove + Remove selected requirements from the environment. + include_build_system + Include build system requirements by using `include`. + name + Name of the environment. + + """ + if remove is None: + remove = set() + if additional_requirements is None: + additional_requirements = [] pip = set(pip) - env = Environment(filename[0], pip_packages=pip, extras=extras, channels=channels) - for f in filename[1:]: - env.combine(Environment(f, pip_packages=pip, extras=extras, channels=channels)) + + env = create_environment( + sources=sources, + additional_requirements=additional_requirements, + channels=channels, + extras=extras, + pip=pip, + ) _include = include_build_system == "include" - env.export(output_file, include_build_system=_include) + env.export(output, include_build_system=_include, remove=remove, name=name) + + +def create_from_definition(env_def: str): + """Create an environment from parameters stored in a definition YAML file. + + Parameters + ---------- + env_def + The definition file. + + """ + with open(env_def, "r") as f: + config = yaml.load(f.read(), yaml.SafeLoader) + create_environment_file(**config) + + +def create_environment( + sources: list[str], + *, + channels: list[str] = None, + extras: list[str] = None, + pip: set[str] = None, + additional_requirements: list[str] = None, +): + """Create an environment instance from multiple source files and additional requirements. + + Parameters + ---------- + sources + The list of source files to combine. + channels + Conda channels to include. + extras + Extras specification to apply to all sources. + pip + List of dependencies to install via pip. + additional_requirements + Additional requirements to include in the environment. + + Returns + ------- + Environment + The environment specification. + """ + if channels is None: + channels = ["conda-forge"] + if extras is None: + extras = [] + if pip is None: + pip = [] + if additional_requirements is None: + additional_requirements = [] + + env = Environment(sources[0], pip_packages=pip, extras=extras, channels=channels) + for source in sources[1:]: + env.combine( + Environment(source, pip_packages=pip, extras=extras, channels=channels) + ) + + env.add_requirements(additional_requirements) + + return env def main(): @@ -30,7 +131,11 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument( - "setup", type=str, nargs="*", default="pyproject.toml", help="dependency file" + "sources", + type=str, + nargs="*", + default="pyproject.toml", + help="dependency files and sources", ) parser.add_argument( "-o", "--output", type=str, default="environment.yml", help="output file" @@ -45,19 +150,25 @@ def main(): choices=["omit", "include"], ) parser.add_argument("-p", "--pip", type=str, nargs="*", default=[]) + parser.add_argument("-r", "--remove", type=str, nargs="*", default=[]) + parser.add_argument( + "-a", "--additional_requirements", type=str, nargs="*", default=[] + ) args = parser.parse_args() - for file in args.setup: + for file in args.sources: filename, _ = split_extras(file) if not Path(filename).is_file(): raise FileNotFoundError(f"Could not find file {filename}") create_environment_file( - filename=args.setup, - output_file=args.output, + sources=args.sources, + output=args.output, channels=args.channels, extras=args.extras, pip=args.pip, + remove=args.remove, + additional_requirements=args.additional_requirements, include_build_system=args.build_system, ) diff --git a/pyproject.toml b/pyproject.toml index 252a175..83db849 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,3 +57,9 @@ find = { exclude = [ [tool.setuptools_scm] write_to = "pydeps2env/_version.py" + +[tool.pytest.ini_options] +addopts = "--tb=short --color=yes -rsw --cov=pydeps2env" +testpaths = [ + "test", +] diff --git a/test/definition.yaml b/test/definition.yaml new file mode 100644 index 0000000..1b9beae --- /dev/null +++ b/test/definition.yaml @@ -0,0 +1,29 @@ +# the target file to create +output: test-configuration.yaml +# default name of the environment +name: test-env +# conda channels to include +channels: +- conda-forge +- defaults +# list of source files that define sub environments +# these will be loaded as Environment() +sources: +- ./test/environment.yaml +- ./test/local.yaml +- ./test/pyproject.toml[doc] +- ./test/requirements.txt +- https://github.com/CagtayFabry/pydeps2env/blob/custom_deps/pyproject.toml +# extras to apply to all sources and packages +extras: +- test +# dependencies that should be removed after collection +remove: +- pyyaml +additional_requirements: +- urllib3 +# include build system dependencies +# list of dependencies that must be pip installed (excluding auto-sorted depedencies like urls) +pip: +- urllib3 +include_build_system: include diff --git a/test/local.yaml b/test/local.yaml new file mode 100644 index 0000000..cca34f3 --- /dev/null +++ b/test/local.yaml @@ -0,0 +1,8 @@ +channels: +- conda-forge +dependencies: +- python>3.7 +- pydeps2env>=1.0.0 +- pip +- pip: + - testproject @ file:/..//test_package diff --git a/test/pyproject.toml b/test/pyproject.toml index 420648b..60e39fb 100644 --- a/test/pyproject.toml +++ b/test/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "ipython", "numpy>=1.20", "pandas>=1", + "pydeps2env @ git+https://github.com/CagtayFabry/pydeps2env.git", ] optional-dependencies.doc = [ "sphinx", diff --git a/test/requirements.txt b/test/requirements.txt index 8a60482..9464df0 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,3 @@ +python>=3.9 setuptools urllib3 diff --git a/test/test_environment.py b/test/test_environment.py new file mode 100644 index 0000000..9dcf7f0 --- /dev/null +++ b/test/test_environment.py @@ -0,0 +1,61 @@ +import pytest +from pydeps2env import Environment, create_environment, create_from_definition + +_inputs = [ + "./test/pyproject.toml[test]", + "./test/setup.cfg[test]", + "./test/requirements.txt", + "./test/environment.yaml", + "./test/local.yaml", + "https://raw.githubusercontent.com/BAMWelDX/weldx/master/pyproject.toml[test]", + "https://github.com/BAMWelDX/weldx/blob/master/pyproject.toml[test]", + "https://raw.githubusercontent.com/BAMWelDX/weldx/v0.3.2/setup.cfg[test]", + # "https://raw.githubusercontent.com/BAMWelDX/weldx/master/doc/rtd_environment.yml", +] + + +class TestEnvironment: + """Test base Environment class.""" + + # test_init ------------------------------------------------------------------------ + + @pytest.mark.parametrize("filename", _inputs) + def test_init( + self, + filename: str, + ): + """Test the `__init__` method of the Environment class.""" + + env = Environment(filename) + assert "python" in env.requirements + if filename.startswith("./test/pyproject.toml"): + assert "pydeps2env" in env.requirements + assert "pydeps2env" in env.pip_packages + + conda, pip = env._get_dependencies() + assert ( + "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git" in pip + ) + + +def test_multiple_sources(): + env = create_environment( + _inputs, + extras=["test"], + pip=["urllib3", "pandas"], + additional_requirements=["k3d"], + ) + + for req in ["python", "pydeps2env", "testproject", "urllib3", "pytest", "k3d"]: + assert req in env.requirements + + for req in ["testproject", "pydeps2env", "requests", "pandas"]: + assert req in env.pip_packages + + conda, pip = env._get_dependencies() + assert "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git" in pip + assert "testproject@ file:/..//test_package" in pip + + +def test_definition(): + create_from_definition("./test/definition.yaml") diff --git a/test/test_package/pyproject.toml b/test/test_package/pyproject.toml new file mode 100644 index 0000000..5da8b22 --- /dev/null +++ b/test/test_package/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = [ + "setuptools>=64", + "setuptools-scm[toml]>=6.2", +] + +[project] +name = "testproject" +version = "0.1.0" +description = "A stub package to test local dependency parsing and installation." +authors = [ + { name = "Çağtay Fabry", email = "cagtay.fabry@bam.de" }, +] +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "bidict", +] diff --git a/test/test_package/testproject/__init__.py b/test/test_package/testproject/__init__.py new file mode 100644 index 0000000..858ba7d --- /dev/null +++ b/test/test_package/testproject/__init__.py @@ -0,0 +1,7 @@ +"""Testproject.""" + +CONST = "A CONSTANT VALUE" + + +def func(): + return CONST