From 74afbe0dc86b5fd963a53d92c3a63b6da84db775 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 26 Mar 2024 13:25:07 +0100
Subject: [PATCH 01/20] support custom dependencies

---
 pydeps2env/__init__.py             |   3 +-
 pydeps2env/environment.py          | 114 ++++++++++++++++++++++-------
 pydeps2env/generate_environment.py |   8 +-
 test/requirements.txt              |   1 +
 4 files changed, 96 insertions(+), 30 deletions(-)

diff --git a/pydeps2env/__init__.py b/pydeps2env/__init__.py
index c3252df..473c7ca 100644
--- a/pydeps2env/__init__.py
+++ b/pydeps2env/__init__.py
@@ -1,8 +1,9 @@
 """pydeps2env: helps to generate conda environment files from python package dependencies."""
 
 from .environment import Environment
+from .generate_environment import create_environment_file
 
-__all__ = ["Environment"]
+__all__ = ["Environment", "create_environment_file"]
 
 try:
     from ._version import __version__
diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 2d3ae71..c23202a 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -1,13 +1,34 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from packaging.requirements import Requirement
+from packaging.requirements import Requirement, InvalidRequirement, ParserSyntaxError
 from pathlib import Path
 from collections import defaultdict
 import configparser
 import tomli as tomllib
 import yaml
 import warnings
+from io import StringIO, BytesIO
+
+
+@dataclass(frozen=True)
+class CustomRequirement:
+    name: str
+
+    def __str__(self) -> str:
+        return self.name
+
+
+def _parse_requirement(req) -> Requirement | CustomRequirement:
+    if isinstance(req, (Requirement, CustomRequirement)):
+        return req
+
+    try:
+        req = Requirement(req)
+    except (InvalidRequirement, ParserSyntaxError, TypeError):
+        req = CustomRequirement(name=req)
+
+    return req
 
 
 def clean_list(item: list, sort: bool = True) -> list:
@@ -32,8 +53,7 @@ def add_requirement(
 ):
     """Add a requirement to existing requirement specification (in place)."""
 
-    if not isinstance(req, Requirement):
-        req = Requirement(req)
+    req = _parse_requirement(req)
 
     if req.name not in requirements:
         requirements[req.name] = req
@@ -71,25 +91,46 @@ def __post_init__(self):
         self.channels = list(dict.fromkeys(self.channels))
         self.pip_packages = set(self.pip_packages)
 
+        # parse and remove extra specs from filename
         if isinstance(self.filename, str):
             self.filename, extras = split_extras(self.filename)
             self.extras |= set(extras)
 
-        if Path(self.filename).suffix == ".toml":
-            self.load_pyproject()
-        elif Path(self.filename).suffix == ".cfg":
-            self.load_config()
-        elif Path(self.filename).suffix in [".yaml", ".yml"]:
-            self.load_yaml()
-        elif Path(self.filename).suffix in [".txt"]:
-            self.load_txt()
+        # store suffix for later parsing
+        self._suffix = Path(self.filename).suffix
+
+        # read file contents into bytes
+        _filename = self.filename
+        if isinstance(_filename, str) and _filename.startswith("http"):
+            import urllib.request
+
+            if "/github.com/" in _filename:
+                _filename = _filename.replace(
+                    "/github.com/", "/raw.githubusercontent.com/"
+                )
+                _filename = _filename.replace("/blob/", "/")
+            with urllib.request.urlopen(_filename) as f:
+                _contents: bytes = f.read()  # read raw content into bytes
+        else:
+            with open(_filename, "rb") as f:
+                _contents: bytes = f.read()
+
+        if self._suffix == ".toml":
+            self.load_pyproject(_contents)
+        elif self._suffix == ".cfg":
+            self.load_config(_contents)
+        elif self._suffix in [".yaml", ".yml"]:
+            self.load_yaml(_contents)
+        elif self._suffix in [".txt"]:
+            self.load_txt(_contents)
         else:
             raise ValueError(f"Unsupported input {self.filename}")
 
-    def load_pyproject(self):
+    def load_pyproject(self, contents: bytes):
         """Load contents from a toml file (assume pyproject.toml layout)."""
-        with open(self.filename, "rb") as fh:
-            cp = defaultdict(dict, tomllib.load(fh))
+
+        tomldict = tomllib.load(BytesIO(contents))
+        cp = defaultdict(dict, tomldict)
 
         if python := cp["project"].get("requires-python"):
             add_requirement("python" + python, self.requirements)
@@ -107,14 +148,15 @@ def load_pyproject(self):
             for dep in extra_deps:
                 add_requirement(dep, self.requirements)
 
-    def load_config(self):
+    def load_config(self, contents: bytes):
         """Load contents from a cfg file (assume setup.cfg layout)."""
         cp = configparser.ConfigParser(
             converters={
                 "list": lambda x: [i.strip() for i in x.split("\n") if i.strip()]
             }
         )
-        cp.read(self.filename)
+
+        cp.read_string(contents.decode("UTF-8"))
 
         if python := cp.get("options", "python_requires"):
             add_requirement("python" + python, self.requirements)
@@ -132,10 +174,9 @@ def load_config(self):
             for dep in extra_deps:
                 add_requirement(dep, self.requirements)
 
-    def load_yaml(self):
+    def load_yaml(self, contents: bytes):
         """Load a conda-style environment.yaml file."""
-        with open(self.filename, "r") as f:
-            env = yaml.load(f.read(), yaml.SafeLoader)
+        env = yaml.load(contents.decode(), yaml.SafeLoader)
 
         self.channels += env.get("channels", [])
         self.channels = list(dict.fromkeys(self.channels))
@@ -146,35 +187,47 @@ def load_yaml(self):
             elif isinstance(dep, dict) and "pip" in dep:
                 add_requirement("pip", self.requirements)
                 for pip_dep in dep["pip"]:
-                    req = Requirement(pip_dep)
+                    req = _parse_requirement(pip_dep)
                     self.pip_packages |= {req.name}
                     add_requirement(req, self.requirements)
 
-    def load_txt(self):
+    def load_txt(self, contents: bytes):
         """Load simple list of requirements from txt file."""
-        with open(self.filename, "r") as f:
-            deps = f.readlines()
+        deps = StringIO(contents.decode()).readlines()
 
         for dep in deps:
             add_requirement(dep, self.requirements)
 
     def _get_dependencies(
-        self, include_build_system: bool = True
+        self,
+        include_build_system: bool = True,
+        remove: list[str] = None,
     ) -> tuple[list[str], list[str]]:
         """Get the default conda environment entries."""
 
+        if remove is None:
+            remove = []
+
         reqs = self.requirements.copy()
         if include_build_system:
             reqs = combine_requirements(reqs, self.build_system)
 
         _python = reqs.pop("python", None)
 
-        deps = [str(r) for r in reqs.values() if r.name not in self.pip_packages]
+        deps = [
+            str(r)
+            for r in reqs.values()
+            if r.name not in self.pip_packages and r.name not in remove
+        ]
         deps.sort(key=str.lower)
         if _python:
             deps = [str(_python)] + deps
 
-        pip = [str(r) for r in reqs.values() if r.name in self.pip_packages]
+        pip = [
+            str(r)
+            for r in reqs.values()
+            if r.name in self.pip_packages and r.name not in remove
+        ]
         pip.sort(key=str.lower)
 
         return deps, pip
@@ -183,8 +236,15 @@ def export(
         self,
         outfile: str | Path = "environment.yaml",
         include_build_system: bool = True,
+        remove: list[str] = None,
     ):
-        deps, pip = self._get_dependencies(include_build_system=include_build_system)
+        """Export the environment to a yaml or txt file."""
+        if remove is None:
+            remove = []
+
+        deps, pip = self._get_dependencies(
+            include_build_system=include_build_system, remove=remove
+        )
 
         conda_env = {"channels": self.channels, "dependencies": deps.copy()}
         if pip:
diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index 07a6154..1e98459 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -14,7 +14,9 @@ def create_environment_file(
     channels: list[str],
     extras: list[str],
     pip: set[str],
-    include_build_system: bool = False,
+    remove: set[str],
+    *,
+    include_build_system: str = "omit",
 ):
     pip = set(pip)
     env = Environment(filename[0], pip_packages=pip, extras=extras, channels=channels)
@@ -22,7 +24,7 @@ def create_environment_file(
         env.combine(Environment(f, pip_packages=pip, extras=extras, channels=channels))
 
     _include = include_build_system == "include"
-    env.export(output_file, include_build_system=_include)
+    env.export(output_file, include_build_system=_include, remove=remove)
 
 
 def main():
@@ -45,6 +47,7 @@ def main():
         choices=["omit", "include"],
     )
     parser.add_argument("-p", "--pip", type=str, nargs="*", default=[])
+    parser.add_argument("-r", "--remove", type=str, nargs="*", default=[])
     args = parser.parse_args()
 
     for file in args.setup:
@@ -59,6 +62,7 @@ def main():
         extras=args.extras,
         pip=args.pip,
         include_build_system=args.build_system,
+        remove=args.remove,
     )
 
 
diff --git a/test/requirements.txt b/test/requirements.txt
index 8a60482..9464df0 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -1,2 +1,3 @@
+python>=3.9
 setuptools
 urllib3

From e15a45c00befb7445828846d889f33df0a6d8180 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 26 Mar 2024 13:25:28 +0100
Subject: [PATCH 02/20] add tests

---
 test/test_environment.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 test/test_environment.py

diff --git a/test/test_environment.py b/test/test_environment.py
new file mode 100644
index 0000000..f8711c2
--- /dev/null
+++ b/test/test_environment.py
@@ -0,0 +1,27 @@
+import pytest
+from pydeps2env import Environment
+
+_inputs = [
+    "./test/pyproject.toml[test]",
+    "./test/setup.cfg[test]",
+    "./test/requirements.txt",
+    "./test/environment.yaml",
+    "https://raw.githubusercontent.com/BAMWelDX/weldx/master/pyproject.toml[test]",
+    "https://github.com/BAMWelDX/weldx/blob/master/pyproject.toml[test]",
+    "https://raw.githubusercontent.com/BAMWelDX/weldx/v0.3.2/setup.cfg[test]",
+    # "https://raw.githubusercontent.com/BAMWelDX/weldx/master/doc/rtd_environment.yml",
+]
+
+class TestEnvironment:
+    """Test base Environment class."""
+    # test_init ------------------------------------------------------------------------
+
+    @pytest.mark.parametrize("filename", _inputs)
+    def test_init(
+        self,
+        filename: str,
+    ):
+        """Test the `__init__` method of the Environment class."""
+
+        env = Environment(filename)
+        assert "python" in env.requirements

From e9a775a9caf15b6e6a66235e6aac6163154ea704 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 26 Mar 2024 13:25:49 +0100
Subject: [PATCH 03/20] fmt

---
 test/test_environment.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_environment.py b/test/test_environment.py
index f8711c2..dc9b3dd 100644
--- a/test/test_environment.py
+++ b/test/test_environment.py
@@ -12,8 +12,10 @@
     # "https://raw.githubusercontent.com/BAMWelDX/weldx/master/doc/rtd_environment.yml",
 ]
 
+
 class TestEnvironment:
     """Test base Environment class."""
+
     # test_init ------------------------------------------------------------------------
 
     @pytest.mark.parametrize("filename", _inputs)

From 8415803bb53f4788a629d34795ae6eee0dc2f3c7 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 26 Mar 2024 15:19:06 +0100
Subject: [PATCH 04/20] add generate_environment function and allow additional
 list of requirements

---
 pydeps2env/__init__.py             |  4 ++--
 pydeps2env/environment.py          | 20 ++++++++++++++++++--
 pydeps2env/generate_environment.py | 27 +++++++++++++++++++++++++++
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/pydeps2env/__init__.py b/pydeps2env/__init__.py
index 473c7ca..bf613de 100644
--- a/pydeps2env/__init__.py
+++ b/pydeps2env/__init__.py
@@ -1,9 +1,9 @@
 """pydeps2env: helps to generate conda environment files from python package dependencies."""
 
 from .environment import Environment
-from .generate_environment import create_environment_file
+from .generate_environment import create_environment, create_environment_file
 
-__all__ = ["Environment", "create_environment_file"]
+__all__ = ["Environment", "create_environment", "create_environment_file"]
 
 try:
     from ._version import __version__
diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index c23202a..a9e2433 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -82,6 +82,7 @@ class Environment:
     channels: list[str] = field(default_factory=lambda: ["conda-forge"])
     extras: set[str] | list[str] = field(default_factory=set)
     pip_packages: set[str] = field(default_factory=set)  # install via pip
+    extra_requirements: list[str] = field(default_factory=list)
     requirements: dict[str, Requirement] = field(default_factory=dict, init=False)
     build_system: dict[str, Requirement] = field(default_factory=dict, init=False)
 
@@ -91,6 +92,18 @@ def __post_init__(self):
         self.channels = list(dict.fromkeys(self.channels))
         self.pip_packages = set(self.pip_packages)
 
+        if self.filename:
+            self._read_source()
+
+    def add_requirements(self, requirements: list[str]):
+        """Manually add a list of additional requirements to the environment."""
+
+        for req in requirements:
+            add_requirement(req, self.requirements)
+
+    def _read_source(self):
+        """Read and parse source definition and add requirements."""
+
         # parse and remove extra specs from filename
         if isinstance(self.filename, str):
             self.filename, extras = split_extras(self.filename)
@@ -217,7 +230,9 @@ def _get_dependencies(
         deps = [
             str(r)
             for r in reqs.values()
-            if r.name not in self.pip_packages and r.name not in remove
+            if isinstance(r, Requirement)
+            and r.name not in self.pip_packages
+            and r.name not in remove
         ]
         deps.sort(key=str.lower)
         if _python:
@@ -226,7 +241,8 @@ def _get_dependencies(
         pip = [
             str(r)
             for r in reqs.values()
-            if r.name in self.pip_packages and r.name not in remove
+            if (not isinstance(r, Requirement) or r.name in self.pip_packages)
+            and r.name not in remove
         ]
         pip.sort(key=str.lower)
 
diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index 1e98459..7b14441 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -27,6 +27,33 @@ def create_environment_file(
     env.export(output_file, include_build_system=_include, remove=remove)
 
 
+def create_environment(
+    sources: list[str],
+    requirements: list[str] = None,
+    channels: list[str] = None,
+    extras: list[str] = None,
+    pip: set[str] = None,
+):
+    if requirements is None:
+        requirements = []
+    if channels is None:
+        channels = ["conda-forge"]
+    if extras is None:
+        extras = []
+    if pip is None:
+        pip = []
+
+    env = Environment(sources[0], pip_packages=pip, extras=extras, channels=channels)
+    for source in sources[1:]:
+        env.combine(
+            Environment(source, pip_packages=pip, extras=extras, channels=channels)
+        )
+
+    env.add_requirements(requirements)
+
+    return env
+
+
 def main():
     import argparse
 

From 90722ac87b243ad5d58786e2dbeaf37349a76428 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Wed, 27 Mar 2024 21:18:33 +0100
Subject: [PATCH 05/20] simplify and add tests

---
 .github/workflows/test.yml | 15 ++++++++++
 pydeps2env/environment.py  | 57 +++++++++++++++++---------------------
 test/pyproject.toml        |  1 +
 test/test_environment.py   |  8 ++++++
 4 files changed, 49 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6eec1da..b258b1c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,6 +1,21 @@
 on: [push, pull_request, workflow_dispatch]
 
 jobs:
+  pytest:
+    name: pytest
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: pip install package
+      run: pip install .[test]
+    - name: run pytest
+      run: pytest
+
   demo_job:
     name: action
     runs-on: ${{ matrix.os }}
diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index a9e2433..5e6d42a 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from packaging.requirements import Requirement, InvalidRequirement, ParserSyntaxError
+from packaging.requirements import Requirement
 from pathlib import Path
 from collections import defaultdict
 import configparser
@@ -9,31 +9,7 @@
 import yaml
 import warnings
 from io import StringIO, BytesIO
-
-
-@dataclass(frozen=True)
-class CustomRequirement:
-    name: str
-
-    def __str__(self) -> str:
-        return self.name
-
-
-def _parse_requirement(req) -> Requirement | CustomRequirement:
-    if isinstance(req, (Requirement, CustomRequirement)):
-        return req
-
-    try:
-        req = Requirement(req)
-    except (InvalidRequirement, ParserSyntaxError, TypeError):
-        req = CustomRequirement(name=req)
-
-    return req
-
-
-def clean_list(item: list, sort: bool = True) -> list:
-    """Remove duplicate entries from a list."""
-    pass
+from warnings import warn
 
 
 def split_extras(filename: str) -> tuple[str, set]:
@@ -53,12 +29,21 @@ def add_requirement(
 ):
     """Add a requirement to existing requirement specification (in place)."""
 
-    req = _parse_requirement(req)
+    if not isinstance(req, Requirement):
+        req = Requirement(req)
 
     if req.name not in requirements:
         requirements[req.name] = req
     elif mode == "combine":
         requirements[req.name].specifier &= req.specifier
+        if req.url:
+            if requirements[req.name].url and requirements[req.name].url != req.url:
+                warn(
+                    f"Replacing url for package {req.name}: {requirements[req.name].url} -> req.url",
+                    UserWarning,
+                    stacklevel=1,
+                )
+                requirements[req.name].url = req.url
     elif mode == "replace":
         requirements[req.name] = req
     else:
@@ -81,7 +66,9 @@ class Environment:
     filename: str | Path
     channels: list[str] = field(default_factory=lambda: ["conda-forge"])
     extras: set[str] | list[str] = field(default_factory=set)
-    pip_packages: set[str] = field(default_factory=set)  # install via pip
+    pip_packages: set[str] = field(
+        default_factory=set
+    )  # names of packages to install via pip
     extra_requirements: list[str] = field(default_factory=list)
     requirements: dict[str, Requirement] = field(default_factory=dict, init=False)
     build_system: dict[str, Requirement] = field(default_factory=dict, init=False)
@@ -95,6 +82,9 @@ def __post_init__(self):
         if self.filename:
             self._read_source()
 
+        # packages with url specification must be pip installed
+        self.pip_packages |= {req.name for req in self.requirements.values() if req.url}
+
     def add_requirements(self, requirements: list[str]):
         """Manually add a list of additional requirements to the environment."""
 
@@ -200,7 +190,7 @@ def load_yaml(self, contents: bytes):
             elif isinstance(dep, dict) and "pip" in dep:
                 add_requirement("pip", self.requirements)
                 for pip_dep in dep["pip"]:
-                    req = _parse_requirement(pip_dep)
+                    req = Requirement(pip_dep)
                     self.pip_packages |= {req.name}
                     add_requirement(req, self.requirements)
 
@@ -209,7 +199,7 @@ def load_txt(self, contents: bytes):
         deps = StringIO(contents.decode()).readlines()
 
         for dep in deps:
-            add_requirement(dep, self.requirements)
+            add_requirement(dep.strip(), self.requirements)
 
     def _get_dependencies(
         self,
@@ -227,11 +217,14 @@ def _get_dependencies(
 
         _python = reqs.pop("python", None)
 
+        _pip_packages = self.pip_packages
+        # _pip_packages |= {r.name for r in reqs.values() if r.url}
+
         deps = [
             str(r)
             for r in reqs.values()
             if isinstance(r, Requirement)
-            and r.name not in self.pip_packages
+            and r.name not in _pip_packages
             and r.name not in remove
         ]
         deps.sort(key=str.lower)
@@ -241,7 +234,7 @@ def _get_dependencies(
         pip = [
             str(r)
             for r in reqs.values()
-            if (not isinstance(r, Requirement) or r.name in self.pip_packages)
+            if (not isinstance(r, Requirement) or r.name in _pip_packages)
             and r.name not in remove
         ]
         pip.sort(key=str.lower)
diff --git a/test/pyproject.toml b/test/pyproject.toml
index d54f4fa..e4fcf06 100644
--- a/test/pyproject.toml
+++ b/test/pyproject.toml
@@ -11,6 +11,7 @@ dependencies = [
   "IPython",
   "numpy>=1.20",
   "pandas>=1",
+  "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git",
 ]
 [project.optional-dependencies]
 doc = [
diff --git a/test/test_environment.py b/test/test_environment.py
index dc9b3dd..6a1f05f 100644
--- a/test/test_environment.py
+++ b/test/test_environment.py
@@ -27,3 +27,11 @@ def test_init(
 
         env = Environment(filename)
         assert "python" in env.requirements
+        if filename.startswith("./test/pyproject.toml"):
+            assert "pydeps2env" in env.requirements
+            assert "pydeps2env" in env.pip_packages
+
+            conda, pip = env._get_dependencies()
+            assert (
+                "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git" in pip
+            )

From 228f1f02b556e02adee022170b7542d337549fba Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Wed, 27 Mar 2024 21:44:40 +0100
Subject: [PATCH 06/20] simplify

---
 pydeps2env/environment.py | 52 +++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 30 deletions(-)

diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 5e6d42a..7b5ca52 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, InitVar
 from packaging.requirements import Requirement
 from pathlib import Path
 from collections import defaultdict
@@ -69,11 +69,11 @@ class Environment:
     pip_packages: set[str] = field(
         default_factory=set
     )  # names of packages to install via pip
-    extra_requirements: list[str] = field(default_factory=list)
+    extra_requirements: InitVar[list[str]] = None
     requirements: dict[str, Requirement] = field(default_factory=dict, init=False)
     build_system: dict[str, Requirement] = field(default_factory=dict, init=False)
 
-    def __post_init__(self):
+    def __post_init__(self, extra_requirements):
         # cleanup duplicates etc.
         self.extras = set(self.extras)
         self.channels = list(dict.fromkeys(self.channels))
@@ -82,15 +82,24 @@ def __post_init__(self):
         if self.filename:
             self._read_source()
 
+        if extra_requirements:
+            self.add_requirements(extra_requirements)
+
         # packages with url specification must be pip installed
         self.pip_packages |= {req.name for req in self.requirements.values() if req.url}
 
     def add_requirements(self, requirements: list[str]):
-        """Manually add a list of additional requirements to the environment."""
+        """Add a list of additional requirements to the environment."""
 
         for req in requirements:
             add_requirement(req, self.requirements)
 
+    def add_build_system(self, requirements: list[str]):
+        """Manually add a list of additional requirements to the build system specification."""
+
+        for req in requirements:
+            add_requirement(req, self.build_system)
+
     def _read_source(self):
         """Read and parse source definition and add requirements."""
 
@@ -136,20 +145,12 @@ def load_pyproject(self, contents: bytes):
         cp = defaultdict(dict, tomldict)
 
         if python := cp["project"].get("requires-python"):
-            add_requirement("python" + python, self.requirements)
-
-        for dep in cp.get("project").get("dependencies"):
-            add_requirement(dep, self.requirements)
-
-        for dep in cp.get("build-system").get("requires"):
-            add_requirement(dep, self.build_system)
+            self.add_requirements(["python" + python])
 
+        self.add_requirements(cp.get("project").get("dependencies"))
+        self.add_build_system(cp.get("build-system").get("requires"))
         for e in self.extras:
-            extra_deps = cp.get("project").get("optional-dependencies").get(e)
-            if not extra_deps:
-                continue
-            for dep in extra_deps:
-                add_requirement(dep, self.requirements)
+            self.add_requirements(cp.get("project").get("optional-dependencies").get(e))
 
     def load_config(self, contents: bytes):
         """Load contents from a cfg file (assume setup.cfg layout)."""
@@ -162,20 +163,12 @@ def load_config(self, contents: bytes):
         cp.read_string(contents.decode("UTF-8"))
 
         if python := cp.get("options", "python_requires"):
-            add_requirement("python" + python, self.requirements)
-
-        for dep in cp.getlist("options", "install_requires"):
-            add_requirement(dep, self.requirements)
-
-        for dep in cp.getlist("options", "setup_requires"):
-            add_requirement(dep, self.build_system)
+            self.add_requirements(["python" + python])
 
+        self.add_requirements(cp.getlist("options", "install_requires"))
+        self.add_build_system(cp.getlist("options", "setup_requires"))
         for e in self.extras:
-            extra_deps = cp.getlist("options.extras_require", e)
-            if not extra_deps:
-                continue
-            for dep in extra_deps:
-                add_requirement(dep, self.requirements)
+            self.add_requirements(cp.getlist("options.extras_require", e))
 
     def load_yaml(self, contents: bytes):
         """Load a conda-style environment.yaml file."""
@@ -198,8 +191,7 @@ def load_txt(self, contents: bytes):
         """Load simple list of requirements from txt file."""
         deps = StringIO(contents.decode()).readlines()
 
-        for dep in deps:
-            add_requirement(dep.strip(), self.requirements)
+        self.add_requirements([dep.strip() for dep in deps])
 
     def _get_dependencies(
         self,

From f8428c7fcd6719090c4b4d21db643ec2a248cf85 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Mon, 8 Apr 2024 23:56:59 +0200
Subject: [PATCH 07/20] refactor

---
 pydeps2env/environment.py          | 12 +++----
 pydeps2env/generate_environment.py | 57 +++++++++++++++++++++++-------
 2 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 7b5ca52..bdee184 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -223,12 +223,12 @@ def _get_dependencies(
         if _python:
             deps = [str(_python)] + deps
 
-        pip = [
-            str(r)
-            for r in reqs.values()
-            if (not isinstance(r, Requirement) or r.name in _pip_packages)
-            and r.name not in remove
-        ]
+        pip = []
+        for r in reqs.values():
+            if (
+                not isinstance(r, Requirement) or r.name in _pip_packages
+            ) and r.name not in remove:
+                pip.append(str(r) if not r.url else r.url)
         pip.sort(key=str.lower)
 
         return deps, pip
diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index 7b14441..ce46b45 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -2,6 +2,8 @@
 
 from pathlib import Path
 
+import yaml
+
 try:
     from pydeps2env.environment import Environment, split_extras
 except ModuleNotFoundError:  # try local file if not installed
@@ -9,22 +11,45 @@
 
 
 def create_environment_file(
-    filename: list[str],
-    output_file: str,
-    channels: list[str],
-    extras: list[str],
-    pip: set[str],
-    remove: set[str],
+    sources: list[str],
+    output: str = "environment.yml",
     *,
+    channels: list[str] = None,
+    extras: list[str] = None,
+    pip: set[str] = None,
+    remove: set[str] = None,
+    additional_requirements: list[str] = None,
     include_build_system: str = "omit",
 ):
+    if channels is None:
+        channels = ["conda-forge"]
+    if extras is None:
+        extras = []
+    if pip is None:
+        pip = []
+    if remove is None:
+        remove = {}
+    if additional_requirements is None:
+        additional_requirements = []
     pip = set(pip)
-    env = Environment(filename[0], pip_packages=pip, extras=extras, channels=channels)
-    for f in filename[1:]:
+    env = Environment(
+        sources[0],
+        pip_packages=pip,
+        extras=extras,
+        channels=channels,
+        extra_requirements=additional_requirements,
+    )
+    for f in sources[1:]:
         env.combine(Environment(f, pip_packages=pip, extras=extras, channels=channels))
 
     _include = include_build_system == "include"
-    env.export(output_file, include_build_system=_include, remove=remove)
+    env.export(output, include_build_system=_include, remove=remove)
+
+
+def create_from_definition(env_def: str):
+    with open(env_def, "r") as f:
+        config = yaml.load(f.read(), yaml.SafeLoader)
+    create_environment_file(**config)
 
 
 def create_environment(
@@ -59,7 +84,11 @@ def main():
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "setup", type=str, nargs="*", default="pyproject.toml", help="dependency file"
+        "sources",
+        type=str,
+        nargs="*",
+        default="pyproject.toml",
+        help="dependency files and sources",
     )
     parser.add_argument(
         "-o", "--output", type=str, default="environment.yml", help="output file"
@@ -75,6 +104,9 @@ def main():
     )
     parser.add_argument("-p", "--pip", type=str, nargs="*", default=[])
     parser.add_argument("-r", "--remove", type=str, nargs="*", default=[])
+    parser.add_argument(
+        "-a", "--additional_requirements", type=str, nargs="*", default=[]
+    )
     args = parser.parse_args()
 
     for file in args.setup:
@@ -83,13 +115,14 @@ def main():
             raise FileNotFoundError(f"Could not find file {filename}")
 
     create_environment_file(
-        filename=args.setup,
+        filename=args.sources,
         output_file=args.output,
         channels=args.channels,
         extras=args.extras,
         pip=args.pip,
-        include_build_system=args.build_system,
         remove=args.remove,
+        additional_requirements=args.additional_requirements,
+        include_build_system=args.build_system,
     )
 
 

From 621d3cfce3133a6fd7aa5ae18f662496e3a766c8 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Fri, 3 May 2024 16:14:52 +0200
Subject: [PATCH 08/20] update create_from_definition

---
 pydeps2env/__init__.py             | 13 +++++++++++--
 pydeps2env/environment.py          | 11 +++++++++--
 pydeps2env/generate_environment.py |  3 ++-
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/pydeps2env/__init__.py b/pydeps2env/__init__.py
index bf613de..ecaa80b 100644
--- a/pydeps2env/__init__.py
+++ b/pydeps2env/__init__.py
@@ -1,9 +1,18 @@
 """pydeps2env: helps to generate conda environment files from python package dependencies."""
 
 from .environment import Environment
-from .generate_environment import create_environment, create_environment_file
+from .generate_environment import (
+    create_environment,
+    create_environment_file,
+    create_from_definition,
+)
 
-__all__ = ["Environment", "create_environment", "create_environment_file"]
+__all__ = [
+    "Environment",
+    "create_environment",
+    "create_environment_file",
+    "create_from_definition",
+]
 
 try:
     from ._version import __version__
diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 5b59460..bfd345c 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -247,6 +247,7 @@ def export(
         outfile: str | Path = "environment.yaml",
         include_build_system: bool = True,
         remove: list[str] = None,
+        name: str = None,
     ):
         """Export the environment to a yaml or txt file."""
         if remove is None:
@@ -256,12 +257,18 @@ def export(
             include_build_system=include_build_system, remove=remove
         )
 
-        conda_env = {"channels": self.channels, "dependencies": deps.copy()}
+        conda_env = {
+            "name": name,
+            "channels": self.channels,
+            "dependencies": deps.copy(),
+        }
         if pip:
             if "pip" not in self.requirements:
                 conda_env["dependencies"] += ["pip"]
             conda_env["dependencies"] += [{"pip": pip}]
 
+        conda_env = {k: v for k, v in conda_env.items() if v}
+
         if outfile is None:
             return conda_env
 
@@ -277,7 +284,7 @@ def export(
                     f"Unknown environment format `{p.suffix}`, generating conda yaml output."
                 )
             with open(p, "w") as outfile:
-                yaml.dump(conda_env, outfile, default_flow_style=False)
+                yaml.dump(conda_env, outfile, default_flow_style=False, sort_keys=False)
 
     def combine(self, other: Environment):
         """Merge other Environment requirements into this Environment."""
diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index ce46b45..6b890f2 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -20,6 +20,7 @@ def create_environment_file(
     remove: set[str] = None,
     additional_requirements: list[str] = None,
     include_build_system: str = "omit",
+    name: str = None,
 ):
     if channels is None:
         channels = ["conda-forge"]
@@ -43,7 +44,7 @@ def create_environment_file(
         env.combine(Environment(f, pip_packages=pip, extras=extras, channels=channels))
 
     _include = include_build_system == "include"
-    env.export(output, include_build_system=_include, remove=remove)
+    env.export(output, include_build_system=_include, remove=remove, name=name)
 
 
 def create_from_definition(env_def: str):

From bcfa993b21567502b20d2a05131221597f7592eb Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 21 May 2024 09:35:22 +0000
Subject: [PATCH 09/20] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 test/pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/pyproject.toml b/test/pyproject.toml
index dd5bf85..60e39fb 100644
--- a/test/pyproject.toml
+++ b/test/pyproject.toml
@@ -19,7 +19,7 @@ dependencies = [
   "ipython",
   "numpy>=1.20",
   "pandas>=1",
-  "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git",
+  "pydeps2env @ git+https://github.com/CagtayFabry/pydeps2env.git",
 ]
 optional-dependencies.doc = [
   "sphinx",

From d209141b40add4f6a9d53ef0df9628ec611ec9ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=87a=C4=9Ftay=20Fabry?= <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 13:12:53 +0200
Subject: [PATCH 10/20] Update generate_environment.py

---
 pydeps2env/generate_environment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index 6b890f2..a8aa924 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -110,7 +110,7 @@ def main():
     )
     args = parser.parse_args()
 
-    for file in args.setup:
+    for file in args.sources:
         filename, _ = split_extras(file)
         if not Path(filename).is_file():
             raise FileNotFoundError(f"Could not find file {filename}")

From b8316b2c84d03772716cb6d3b207d7fc91f393ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=87a=C4=9Ftay=20Fabry?= <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 13:17:07 +0200
Subject: [PATCH 11/20] Update generate_environment.py

---
 pydeps2env/generate_environment.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index a8aa924..7f45b24 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -116,8 +116,8 @@ def main():
             raise FileNotFoundError(f"Could not find file {filename}")
 
     create_environment_file(
-        filename=args.sources,
-        output_file=args.output,
+        sources=args.sources,
+        output=args.output,
         channels=args.channels,
         extras=args.extras,
         pip=args.pip,

From a46462ba2efaeba4004c0612b93b41170fa0ef0f Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 14:12:26 +0200
Subject: [PATCH 12/20] use package name in pip requirements again

a relative local directory has du be indicated like file:/../directory
---
 pydeps2env/environment.py | 2 +-
 pyproject.toml            | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index bfd345c..07acb73 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -237,7 +237,7 @@ def _get_dependencies(
             if (
                 not isinstance(r, Requirement) or r.name in _pip_packages
             ) and r.name not in remove:
-                pip.append(str(r) if not r.url else r.url)
+                pip.append(str(r))
         pip.sort(key=str.lower)
 
         return deps, pip
diff --git a/pyproject.toml b/pyproject.toml
index 252a175..83db849 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,3 +57,9 @@ find = { exclude = [
 
 [tool.setuptools_scm]
 write_to = "pydeps2env/_version.py"
+
+[tool.pytest.ini_options]
+addopts = "--tb=short --color=yes -rsw --cov=pydeps2env"
+testpaths = [
+  "test",
+]

From aba23cbf2712a8272e22469359858ef0b06ef2bc Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 17:04:11 +0200
Subject: [PATCH 13/20] update tests and local pip handling

---
 .gitignore                                | 167 ++++++++++++++++++++++
 README.md                                 |  31 +++-
 action.yml                                |  15 +-
 pydeps2env/environment.py                 |   2 +-
 test/local.yaml                           |   8 ++
 test/test_environment.py                  |  17 ++-
 test/test_package/pyproject.toml          |  31 ++++
 test/test_package/testproject/__init__.py |   7 +
 8 files changed, 265 insertions(+), 13 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 test/local.yaml
 create mode 100644 test/test_package/pyproject.toml
 create mode 100644 test/test_package/testproject/__init__.py

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..e0ace01
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,167 @@
+# custom folders
+demo/
+
+pydeps2env/_version.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
diff --git a/README.md b/README.md
index 8cd04e3..25900cb 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 # pydeps2env
 
 An easy way to create conda environment files from you python project dependencies.  
-Creates a conda `environment.yml` file from python package dependencies listed in a `pyproject.toml` or `setup.cfg` file.
+Creates a conda `environment.yml` file from python package dependencies listed in on or multiple different source formats like `pyproject.toml`, `setup.cfg`, `environment.yaml` or `requirements.txt` files.
 
 The project contains
 - GitHub action
@@ -42,7 +42,7 @@ test = ["pytest"]
 pip_only = ["bidict"]
 ```
 
-The default parameters will output this sorted `environment.yml` (note that the `python` dependency will always be the first item on the list):
+The default parameters will output this sorted `environment.yml` (note that the `python` dependency specification will always be the first item on the list):
 
 ```yaml
 channels:
@@ -74,13 +74,33 @@ dependencies:
     - bidict
 ```
 
-## configuration options
+## basic usage (python)
+
+Create an `Environment` using Python and export it to an `environment.yaml` file.
+
+```python
+from pydeps2env import Environment
+
+env = Environment("./test/pyproject.toml[doc]")
+env.export("my_environment.yaml")
+```
+
+## basic usage (command line)
+
+Combine multiple source files into a single environment file (including build dependencies).
+Install pandas using `pip`.
+
+```bash
+pydeps2env ./test/setup.cfg ./test/pyproject.toml[doc] ./test/environment.yaml ./test/requirements.txt -o output.yaml -c defaults --extras test -b include --pip pandas
+```
+
+## configuration options (GitHub action)
 
 To customize the output the input options are available to the action:
 
 ### files
 
-Specify the location of the `'setup.cfg'` or `'pyproject.toml'` files to parse. (defaults to `'pyproject.toml'`)
+Specify the location of the dependencies files to parse. (defaults to 'pyproject.toml')
 Multiple files can be listed. This will result in a combined environment file.
 
 ### output:
@@ -105,7 +125,8 @@ is `'omit'` so no setup dependencies will be installed).
 
 ### pip
 List of packages to install via `pip` instead of `conda`.
-The dependencies will be listet under the `pip:` section in the environment file.
+The dependencies will be listed under the `pip:` section in the environment file.
+If a dependency is listed with its URN, it will always be installed via pip (e.g. `pydeps2env @ git+https://github.com/CagtayFabry/pydeps2env`)
 
 ## example
 
diff --git a/action.yml b/action.yml
index ac0d34c..2d21d56 100644
--- a/action.yml
+++ b/action.yml
@@ -7,6 +7,7 @@ inputs:
   files:
     description: >-
       Specify the location of the dependencies files to parse. (defaults to 'pyproject.toml')
+      Separate a list of multiple source files by spaces (e.g. 'pyproject.toml[test] requirements.txt').
     required: true
     default: pyproject.toml
   output:
@@ -16,25 +17,27 @@ inputs:
     default: environment.yml
   channels:
     description: >-
-      List the conda channels to include in the environment file. (defaults to 'defaults')
+      List the conda channels to include in the environment file. (defaults to 'conda-forge')
       Separate a list of multiple channels by spaces (e.g. 'conda-forge defaults').
     required: true
-    default: defaults
+    default: conda-forge
   extras:
     description: >-
-      Specify one or more optional [extras_require] sections to add to the environment
-      (e.g. 'test' to include package that you would normally install with 'pip install pkg[test]')
+      Specify one or more optional [extras_require] sections to add to all source files (s.a.).
+      (e.g. 'test' to include package that you would normally install with 'pip install pkg[test]').
+      Note that for individual sources in 'files', the '[extra]' syntax is also possible.
     required: false
   build_system:
     description: >-
-      if set to 'include' the dependencies listed under [build-system] or [options]:setup_requires will be added to the environment
-      (default is 'omit' so no setup dependencies will be installed)
+      if set to 'include' the dependencies listed under [build-system] (for 'pyproject.toml') or [options]:setup_requires (for 'setup.cfg') will be added to the environment
+      (default is 'omit' so no build system dependencies will be installed)
     required: false
     default: omit
   pip:
     description: >-
       List of packages to install via pip instead of conda.
       The dependencies will be listet under the pip section in the environment file.
+      If a dependency is listed with its URN, it will always be installed via pip (e.g. 'pydeps2env @ git+https://github.com/CagtayFabry/pydeps2env')
     required: false
 runs:
   using: composite
diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 07acb73..4cf46f8 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -237,7 +237,7 @@ def _get_dependencies(
             if (
                 not isinstance(r, Requirement) or r.name in _pip_packages
             ) and r.name not in remove:
-                pip.append(str(r))
+                pip.append(str(r) if not r.url else f"{r.name}@ {r.url}")
         pip.sort(key=str.lower)
 
         return deps, pip
diff --git a/test/local.yaml b/test/local.yaml
new file mode 100644
index 0000000..cca34f3
--- /dev/null
+++ b/test/local.yaml
@@ -0,0 +1,8 @@
+channels:
+- conda-forge
+dependencies:
+- python>3.7
+- pydeps2env>=1.0.0
+- pip
+- pip:
+  - testproject @ file:/..//test_package
diff --git a/test/test_environment.py b/test/test_environment.py
index 6a1f05f..06be47b 100644
--- a/test/test_environment.py
+++ b/test/test_environment.py
@@ -1,11 +1,12 @@
 import pytest
-from pydeps2env import Environment
+from pydeps2env import Environment, create_environment
 
 _inputs = [
     "./test/pyproject.toml[test]",
     "./test/setup.cfg[test]",
     "./test/requirements.txt",
     "./test/environment.yaml",
+    "./test/local.yaml",
     "https://raw.githubusercontent.com/BAMWelDX/weldx/master/pyproject.toml[test]",
     "https://github.com/BAMWelDX/weldx/blob/master/pyproject.toml[test]",
     "https://raw.githubusercontent.com/BAMWelDX/weldx/v0.3.2/setup.cfg[test]",
@@ -35,3 +36,17 @@ def test_init(
             assert (
                 "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git" in pip
             )
+
+
+def test_multiple_sources():
+    env = create_environment(_inputs, extras=["test"], pip=["urllib3", "pandas"])
+
+    for req in ["python", "pydeps2env", "testproject", "urllib3", "pytest"]:
+        assert req in env.requirements
+
+    for req in ["testproject", "pydeps2env", "requests", "pandas"]:
+        assert req in env.pip_packages
+
+    conda, pip = env._get_dependencies()
+    assert "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git" in pip
+    assert "testproject@ file:/..//test_package" in pip
diff --git a/test/test_package/pyproject.toml b/test/test_package/pyproject.toml
new file mode 100644
index 0000000..5da8b22
--- /dev/null
+++ b/test/test_package/pyproject.toml
@@ -0,0 +1,31 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = [
+  "setuptools>=64",
+  "setuptools-scm[toml]>=6.2",
+]
+
+[project]
+name = "testproject"
+version = "0.1.0"
+description = "A stub package to test local dependency parsing and installation."
+authors = [
+  { name = "Çağtay Fabry", email = "cagtay.fabry@bam.de" },
+]
+requires-python = ">=3.9"
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: BSD License",
+  "Natural Language :: English",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+  "bidict",
+]
diff --git a/test/test_package/testproject/__init__.py b/test/test_package/testproject/__init__.py
new file mode 100644
index 0000000..858ba7d
--- /dev/null
+++ b/test/test_package/testproject/__init__.py
@@ -0,0 +1,7 @@
+"""Testproject."""
+
+CONST = "A CONSTANT VALUE"
+
+
+def func():
+    return CONST

From 36684086adc9b11dc3e760c9208c4e336b8876e8 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 17:15:47 +0200
Subject: [PATCH 14/20] simplify

---
 pydeps2env/generate_environment.py | 30 ++++++++++++------------------
 test/test_environment.py           | 11 ++++++++---
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index 7f45b24..4a3c291 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -17,31 +17,24 @@ def create_environment_file(
     channels: list[str] = None,
     extras: list[str] = None,
     pip: set[str] = None,
-    remove: set[str] = None,
     additional_requirements: list[str] = None,
+    remove: set[str] = None,
     include_build_system: str = "omit",
     name: str = None,
 ):
-    if channels is None:
-        channels = ["conda-forge"]
-    if extras is None:
-        extras = []
-    if pip is None:
-        pip = []
     if remove is None:
         remove = {}
     if additional_requirements is None:
         additional_requirements = []
     pip = set(pip)
-    env = Environment(
-        sources[0],
-        pip_packages=pip,
-        extras=extras,
+
+    env = create_environment(
+        sources=sources,
+        requirements=additional_requirements,
         channels=channels,
-        extra_requirements=additional_requirements,
+        extras=extras,
+        pip=pip,
     )
-    for f in sources[1:]:
-        env.combine(Environment(f, pip_packages=pip, extras=extras, channels=channels))
 
     _include = include_build_system == "include"
     env.export(output, include_build_system=_include, remove=remove, name=name)
@@ -55,19 +48,20 @@ def create_from_definition(env_def: str):
 
 def create_environment(
     sources: list[str],
-    requirements: list[str] = None,
+    *,
     channels: list[str] = None,
     extras: list[str] = None,
     pip: set[str] = None,
+    additional_requirements: list[str] = None,
 ):
-    if requirements is None:
-        requirements = []
     if channels is None:
         channels = ["conda-forge"]
     if extras is None:
         extras = []
     if pip is None:
         pip = []
+    if additional_requirements is None:
+        additional_requirements = []
 
     env = Environment(sources[0], pip_packages=pip, extras=extras, channels=channels)
     for source in sources[1:]:
@@ -75,7 +69,7 @@ def create_environment(
             Environment(source, pip_packages=pip, extras=extras, channels=channels)
         )
 
-    env.add_requirements(requirements)
+    env.add_requirements(additional_requirements)
 
     return env
 
diff --git a/test/test_environment.py b/test/test_environment.py
index 06be47b..e0b5ebc 100644
--- a/test/test_environment.py
+++ b/test/test_environment.py
@@ -39,9 +39,14 @@ def test_init(
 
 
 def test_multiple_sources():
-    env = create_environment(_inputs, extras=["test"], pip=["urllib3", "pandas"])
-
-    for req in ["python", "pydeps2env", "testproject", "urllib3", "pytest"]:
+    env = create_environment(
+        _inputs,
+        extras=["test"],
+        pip=["urllib3", "pandas"],
+        additional_requirements=["k3d"],
+    )
+
+    for req in ["python", "pydeps2env", "testproject", "urllib3", "pytest", "k3d"]:
         assert req in env.requirements
 
     for req in ["testproject", "pydeps2env", "requests", "pandas"]:

From 497b3331b558f13b64e72e930e086b17b5647f7e Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com>
Date: Tue, 21 May 2024 17:33:24 +0200
Subject: [PATCH 15/20] add docstrings

---
 pydeps2env/environment.py          |  2 +-
 pydeps2env/generate_environment.py | 54 +++++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 4cf46f8..50ea583 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -27,7 +27,7 @@ def split_extras(filename: str) -> tuple[str, set]:
         filename, extras = filename.split("[", 1)
         extras = set(extras.split("]", 1)[0].split(","))
     else:
-        extras = {}
+        extras = set()
     return filename, extras
 
 
diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index 4a3c291..1c2d98d 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -22,8 +22,32 @@ def create_environment_file(
     include_build_system: str = "omit",
     name: str = None,
 ):
+    """Create an environment file from multiple source files and additional requirements.
+
+    Parameters
+    ----------
+    sources
+        The list of source files to combine.
+    output
+        The output filename to generate
+    channels
+        Conda channels to include.
+    extras
+        Extras specification to apply to all sources.
+    pip
+        List of dependencies to install via pip.
+    additional_requirements
+        Additional requirements to include in the environment.
+    remove
+        Remove selected requirements from the environment.
+    include_build_system
+        Include build system requirements by using `include`.
+    name
+        Name of the environment.
+
+    """
     if remove is None:
-        remove = {}
+        remove = set()
     if additional_requirements is None:
         additional_requirements = []
     pip = set(pip)
@@ -41,6 +65,14 @@ def create_environment_file(
 
 
 def create_from_definition(env_def: str):
+    """Create an environment from parameters stored in a definition YAML file.
+
+    Parameters
+    ----------
+    env_def
+        The definition file.
+
+    """
     with open(env_def, "r") as f:
         config = yaml.load(f.read(), yaml.SafeLoader)
     create_environment_file(**config)
@@ -54,6 +86,26 @@ def create_environment(
     pip: set[str] = None,
     additional_requirements: list[str] = None,
 ):
+    """Create an environment instance from multiple source files and additional requirements.
+
+    Parameters
+    ----------
+    sources
+        The list of source files to combine.
+    channels
+        Conda channels to include.
+    extras
+        Extras specification to apply to all sources.
+    pip
+        List of dependencies to install via pip.
+    additional_requirements
+        Additional requirements to include in the environment.
+
+    Returns
+    -------
+    Environment
+        The environment specification.
+    """
     if channels is None:
         channels = ["conda-forge"]
     if extras is None:

From 4ee37422070391f35363dea78e82abb57fb5d76f Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 17:38:41 +0200
Subject: [PATCH 16/20] fix

---
 pydeps2env/generate_environment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py
index 1c2d98d..2c97ed9 100644
--- a/pydeps2env/generate_environment.py
+++ b/pydeps2env/generate_environment.py
@@ -54,7 +54,7 @@ def create_environment_file(
 
     env = create_environment(
         sources=sources,
-        requirements=additional_requirements,
+        additional_requirements=additional_requirements,
         channels=channels,
         extras=extras,
         pip=pip,

From d0e86ccfc70ca606adf035ef82bc0b7604e410a2 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 17:49:50 +0200
Subject: [PATCH 17/20] fix pip formatting

---
 pydeps2env/environment.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 50ea583..5cc08e7 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -224,7 +224,7 @@ def _get_dependencies(
         deps = [
             str(r)
             for r in reqs.values()
-            if isinstance(r, Requirement)
+            if not r.url  # install via pip
             and r.name not in _pip_packages
             and r.name not in remove
         ]
@@ -232,12 +232,13 @@ def _get_dependencies(
         if _python:
             deps = [str(_python)] + deps
 
-        pip = []
-        for r in reqs.values():
-            if (
-                not isinstance(r, Requirement) or r.name in _pip_packages
-            ) and r.name not in remove:
-                pip.append(str(r) if not r.url else f"{r.name}@ {r.url}")
+        pip = [
+            r
+            for r in reqs.values()
+            if (r.name in _pip_packages or r.url) and r.name not in remove
+        ]
+        # string formatting
+        pip = [str(r) if not r.url else f"{r.name}@ {r.url}" for r in pip]
         pip.sort(key=str.lower)
 
         return deps, pip

From 20b60716d07be048a22cd64a61a69d328a96f8bc Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 18:33:09 +0200
Subject: [PATCH 18/20] ad definition test

---
 pydeps2env/environment.py | 1 +
 test/test_environment.py  | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py
index 5cc08e7..c6d7f56 100644
--- a/pydeps2env/environment.py
+++ b/pydeps2env/environment.py
@@ -83,6 +83,7 @@ class Environment:
     build_system: dict[str, Requirement] = field(default_factory=dict, init=False)
 
     def __post_init__(self, extra_requirements):
+        print(self.filename)
         # cleanup duplicates etc.
         self.extras = set(self.extras)
         self.channels = list(dict.fromkeys(self.channels))
diff --git a/test/test_environment.py b/test/test_environment.py
index e0b5ebc..9dcf7f0 100644
--- a/test/test_environment.py
+++ b/test/test_environment.py
@@ -1,5 +1,5 @@
 import pytest
-from pydeps2env import Environment, create_environment
+from pydeps2env import Environment, create_environment, create_from_definition
 
 _inputs = [
     "./test/pyproject.toml[test]",
@@ -55,3 +55,7 @@ def test_multiple_sources():
     conda, pip = env._get_dependencies()
     assert "pydeps2env@ git+https://github.com/CagtayFabry/pydeps2env.git" in pip
     assert "testproject@ file:/..//test_package" in pip
+
+
+def test_definition():
+    create_from_definition("./test/definition.yaml")

From de433d41457fa80a515708c881e94f04788f8d39 Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 18:34:48 +0200
Subject: [PATCH 19/20] update readme

---
 README.md | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/README.md b/README.md
index 25900cb..b6cc488 100644
--- a/README.md
+++ b/README.md
@@ -94,6 +94,44 @@ Install pandas using `pip`.
 pydeps2env ./test/setup.cfg ./test/pyproject.toml[doc] ./test/environment.yaml ./test/requirements.txt -o output.yaml -c defaults --extras test -b include --pip pandas
 ```
 
+## advanced usage (definition file)
+
+Users can store complex configurations in a single yaml file and create the desired output using `create_from_definition`.
+Example definition file:
+
+```yaml
+# the target file to create
+output: test-configuration.yaml
+# default name of the environment
+name: test-env
+# conda channels to include
+channels:
+- conda-forge
+- defaults
+# list of source files that define sub environments
+# these will be loaded as Environment()
+sources:
+- ./test/environment.yaml
+- ./test/local.yaml
+- ./test/pyproject.toml[doc]
+- ./test/requirements.txt
+- https://github.com/CagtayFabry/pydeps2env/blob/custom_deps/pyproject.toml
+# extras to apply to all sources and packages
+extras:
+- test
+# dependencies that should be removed after collection
+remove:
+- pyyaml
+additional_requirements:
+- urllib3
+# include build system dependencies
+# list of dependencies that must be pip installed (excluding auto-sorted depedencies like urls)
+pip:
+- urllib3
+include_build_system: include
+
+```
+
 ## configuration options (GitHub action)
 
 To customize the output the input options are available to the action:

From 1231c4719dfea72117d654050c8911eacc771a1f Mon Sep 17 00:00:00 2001
From: Cagtay Fabry <cagtay.fabry@bam.de>
Date: Tue, 21 May 2024 20:17:39 +0200
Subject: [PATCH 20/20] add missing file

---
 test/definition.yaml | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 test/definition.yaml

diff --git a/test/definition.yaml b/test/definition.yaml
new file mode 100644
index 0000000..1b9beae
--- /dev/null
+++ b/test/definition.yaml
@@ -0,0 +1,29 @@
+# the target file to create
+output: test-configuration.yaml
+# default name of the environment
+name: test-env
+# conda channels to include
+channels:
+- conda-forge
+- defaults
+# list of source files that define sub environments
+# these will be loaded as Environment()
+sources:
+- ./test/environment.yaml
+- ./test/local.yaml
+- ./test/pyproject.toml[doc]
+- ./test/requirements.txt
+- https://github.com/CagtayFabry/pydeps2env/blob/custom_deps/pyproject.toml
+# extras to apply to all sources and packages
+extras:
+- test
+# dependencies that should be removed after collection
+remove:
+- pyyaml
+additional_requirements:
+- urllib3
+# include build system dependencies
+# list of dependencies that must be pip installed (excluding auto-sorted depedencies like urls)
+pip:
+- urllib3
+include_build_system: include