diff --git a/MANIFEST.in b/MANIFEST.in index b1fc69e..e69de29 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +0,0 @@ -include VERSION \ No newline at end of file diff --git a/Makefile b/Makefile index 75ba49c..806ba7f 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,21 @@ .venv: pyproject.toml - python -m virtualenv .venv + python3 -m venv .venv -.venv/deps: .venv pyproject.toml setup.cfg - .venv/bin/python -m pip install . build pytest twine +.venv/deps: .venv pyproject.toml + .venv/bin/python -m pip install .[dev] touch .venv/deps build: .venv/deps rm -rf ./dist/ - .venv/bin/python -m build . + .venv/bin/python -m build # only works with python 3+ lint: .venv/deps - .venv/bin/python -m pip install black==22.3.0 - .venv/bin/python -m black --check . + .venv/bin/validate-pyproject pyproject.toml + .venv/bin/black --check deepmerge + .venv/bin/mypy deepmerge test: .venv/deps - .venv/bin/python -m pytest deepmerge + .venv/bin/pytest deepmerge -ready-pr: test lint \ No newline at end of file +ready-pr: test lint diff --git a/README.rst b/README.rst index 24050eb..c8b61c5 100644 --- a/README.rst +++ b/README.rst @@ -2,17 +2,28 @@ deepmerge ========= +.. image:: https://img.shields.io/pypi/v/deepmerge.svg + :target: https://pypi.org/project/deepmerge/ + +.. image:: https://img.shields.io/pypi/status/deepmerge.svg + :target: https://pypi.org/project/deepmerge/ + +.. image:: https://img.shields.io/pypi/pyversions/pillar.svg + :target: https://github.com/toumorokoshi/deepmerge + +.. image:: https://img.shields.io/github/license/toumorokoshi/deepmerge.svg + :target: https://github.com/toumorokoshi/deepmerge + .. image:: https://github.com/toumorokoshi/deepmerge/actions/workflows/python-package.yaml/badge.svg :target: https://github.com/toumorokoshi/deepmerge/actions/workflows/python-package.yaml -A tools to handle merging of -nested data structures in python. +A tool to handle merging of nested data structures in Python. ------------ Installation ------------ -deepmerge is available on `pypi `_: +deepmerge is available on `pypi `_: .. code-block:: bash @@ -67,4 +78,15 @@ Example You can also pass in your own merge functions, instead of a string. -For more information, see the `docs `_ \ No newline at end of file +For more information, see the `docs `_ + +------------------ +Supported Versions +------------------ + +deepmerge is supported on Python 3.8+. + +For older Python versions the last supported version of deepmerge is listed +below: + +- 3.7 : 1.1.1 diff --git a/deepmerge/__init__.py b/deepmerge/__init__.py index b7686cf..503089c 100644 --- a/deepmerge/__init__.py +++ b/deepmerge/__init__.py @@ -1,9 +1,11 @@ +from __future__ import annotations + from .merger import Merger from .strategy.core import STRATEGY_END # noqa # some standard mergers available -DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES = [ +DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES: list[tuple[type, str]] = [ (list, "append"), (dict, "merge"), (set, "union"), @@ -13,15 +15,13 @@ # in the case of type mismatches, # the value from the second object # will override the previous one. -always_merger = Merger( - DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["override"], ["override"] -) +always_merger: Merger = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["override"], ["override"]) # this merge strategies attempts # to merge (append for list, unify for dicts) # if possible, but raises an exception # in the case of type conflicts. -merge_or_raise = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, [], []) +merge_or_raise: Merger = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, [], []) # a conservative merge tactic: # for data structures with a specific @@ -29,6 +29,6 @@ # similar to always_merger but instead # keeps existing values when faced # with a type conflict. -conservative_merger = Merger( +conservative_merger: Merger = Merger( DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["use_existing"], ["use_existing"] ) diff --git a/deepmerge/compat.py b/deepmerge/compat.py deleted file mode 100644 index d2872a6..0000000 --- a/deepmerge/compat.py +++ /dev/null @@ -1,4 +0,0 @@ -try: - string_type = basestring -except: - string_type = str diff --git a/deepmerge/exception.py b/deepmerge/exception.py index ea36dac..c266c09 100644 --- a/deepmerge/exception.py +++ b/deepmerge/exception.py @@ -1,18 +1,35 @@ +from __future__ import annotations + +from typing import Any + +import deepmerge.merger + + class DeepMergeException(Exception): - pass + "Base class for all `deepmerge` Exceptions" class StrategyNotFound(DeepMergeException): - pass + "Exception for when a strategy cannot be located" class InvalidMerge(DeepMergeException): - def __init__(self, strategy_list_name, merge_args, merge_kwargs): - super(InvalidMerge, self).__init__( - "no more strategies found for {0} and arguments {1}, {2}".format( - strategy_list_name, merge_args, merge_kwargs - ) + "Exception for when unable to complete a merge operation" + + def __init__( + self, + strategy_list_name: str, + config: deepmerge.merger.Merger, + path: list, + base: Any, + nxt: Any, + ) -> None: + super().__init__( + f"Could not merge using {strategy_list_name!r} [{config=}, {path=}, {base=}, {nxt=}]" ) self.strategy_list_name = strategy_list_name - self.merge_args = merge_args - self.merge_kwargs = merge_kwargs + self.config = config + self.path = path + self.base = base + self.nxt = nxt + return diff --git a/deepmerge/extended_set.py b/deepmerge/extended_set.py index 1d51b43..3efb399 100644 --- a/deepmerge/extended_set.py +++ b/deepmerge/extended_set.py @@ -1,3 +1,6 @@ +from typing import Sequence, Any + + class ExtendedSet(set): """ ExtendedSet is an extension of set, which allows for usage @@ -9,17 +12,17 @@ class ExtendedSet(set): - unhashable types """ - def __init__(self, elements): - self._values_by_hash = {self._hash(e): e for e in elements} + def __init__(self, elements: Sequence) -> None: + self._values_by_hash = {self._hash_element(e): e for e in elements} - def _insert(self, element): - self._values_by_hash[self._hash(element)] = element + def _insert(self, element: Any) -> None: + self._values_by_hash[self._hash_element(element)] = element + return - def _hash(self, element): + def _hash_element(self, element: Any) -> int: if getattr(element, "__hash__") is not None: return hash(element) - else: - return hash(str(element)) + return hash(str(element)) - def __contains__(self, obj): - return self._hash(obj) in self._values_by_hash + def __contains__(self, obj: Any) -> bool: + return self._hash_element(obj) in self._values_by_hash diff --git a/deepmerge/merger.py b/deepmerge/merger.py index 5ccd568..6c85ff9 100644 --- a/deepmerge/merger.py +++ b/deepmerge/merger.py @@ -1,44 +1,60 @@ -from .strategy.list import ListStrategies -from .strategy.dict import DictStrategies -from .strategy.set import SetStrategies -from .strategy.type_conflict import TypeConflictStrategies -from .strategy.fallback import FallbackStrategies +from __future__ import annotations +from typing import Any, Sequence, Callable -class Merger(object): +from . import strategy as s + + +class Merger: """ + Merges objects based on provided strategies + :param type_strategies, List[Tuple]: a list of (Type, Strategy) pairs that should be used against incoming types. For example: (dict, "override"). """ - PROVIDED_TYPE_STRATEGIES = { - list: ListStrategies, - dict: DictStrategies, - set: SetStrategies, + PROVIDED_TYPE_STRATEGIES: dict[type, type[s.StrategyList]] = { + list: s.ListStrategies, + dict: s.DictStrategies, + set: s.SetStrategies, } - def __init__(self, type_strategies, fallback_strategies, type_conflict_strategies): - self._fallback_strategy = FallbackStrategies(fallback_strategies) + def __init__( + self, + type_strategies: Sequence[tuple[type, s.StrategyCallable | s.StrategyListInitable]], + fallback_strategies: s.StrategyListInitable, + type_conflict_strategies: s.StrategyListInitable, + ) -> None: + self._fallback_strategy = s.FallbackStrategies(fallback_strategies) + self._type_conflict_strategy = s.TypeConflictStrategies(type_conflict_strategies) - expanded_type_strategies = [] + self._type_strategies: list[tuple[type, s.StrategyCallable]] = [] for typ, strategy in type_strategies: if typ in self.PROVIDED_TYPE_STRATEGIES: - strategy = self.PROVIDED_TYPE_STRATEGIES[typ](strategy) - expanded_type_strategies.append((typ, strategy)) - self._type_strategies = expanded_type_strategies - - self._type_conflict_strategy = TypeConflictStrategies(type_conflict_strategies) - - def merge(self, base, nxt): + # Customise a StrategyList instance for this type + self._type_strategies.append((typ, self.PROVIDED_TYPE_STRATEGIES[typ](strategy))) + elif callable(strategy): + self._type_strategies.append((typ, strategy)) + else: + raise ValueError(f"Cannot handle ({typ}, {strategy})") + return + + def merge(self, base: Any, nxt: Any) -> Any: return self.value_strategy([], base, nxt) - def type_conflict_strategy(self, *args): - return self._type_conflict_strategy(self, *args) + def type_conflict_strategy(self, path: list, base: Any, nxt: Any) -> Any: + return self._type_conflict_strategy(self, path, base, nxt) - def value_strategy(self, path, base, nxt): + def value_strategy(self, path: list, base: Any, nxt: Any) -> Any: + # Check for strategy based on type of base, next for typ, strategy in self._type_strategies: if isinstance(base, typ) and isinstance(nxt, typ): + # We have a strategy for this type return strategy(self, path, base, nxt) - if not (isinstance(base, type(nxt)) or isinstance(nxt, type(base))): - return self.type_conflict_strategy(path, base, nxt) - return self._fallback_strategy(self, path, base, nxt) + + if isinstance(base, type(nxt)) or isinstance(nxt, type(base)): + # no known strategy but base, next are similar types + return self._fallback_strategy(self, path, base, nxt) + + # No known strategy and base, next are different types. + return self.type_conflict_strategy(path, base, nxt) diff --git a/deepmerge/py.typed b/deepmerge/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/deepmerge/strategy/__init__.py b/deepmerge/strategy/__init__.py index e69de29..d62ea7c 100644 --- a/deepmerge/strategy/__init__.py +++ b/deepmerge/strategy/__init__.py @@ -0,0 +1,6 @@ +from .core import StrategyList, StrategyCallable, StrategyListInitable +from .dict import DictStrategies +from .fallback import FallbackStrategies +from .list import ListStrategies +from .set import SetStrategies +from .type_conflict import TypeConflictStrategies diff --git a/deepmerge/strategy/core.py b/deepmerge/strategy/core.py index 91fcfbb..b035f57 100644 --- a/deepmerge/strategy/core.py +++ b/deepmerge/strategy/core.py @@ -1,20 +1,33 @@ +from __future__ import annotations + +import sys +from typing import Callable, Any + +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + +import deepmerge.merger from ..exception import StrategyNotFound, InvalidMerge -from ..compat import string_type STRATEGY_END = object() +# Note: We use string annotations here to prevent circular import caused by Merger +StrategyCallable: TypeAlias = "Callable[[deepmerge.merger.Merger, list, Any, Any], Any]" +StrategyListInitable: TypeAlias = "str | StrategyCallable | list[str | StrategyCallable]" -class StrategyList(object): - NAME = None +class StrategyList: + NAME: str - def __init__(self, strategy_list): + def __init__(self, strategy_list: StrategyListInitable) -> None: if not isinstance(strategy_list, list): strategy_list = [strategy_list] - self._strategies = [self._expand_strategy(s) for s in strategy_list] + self._strategies: list[StrategyCallable] = [self._expand_strategy(s) for s in strategy_list] @classmethod - def _expand_strategy(cls, strategy): + def _expand_strategy(cls, strategy: str | StrategyCallable) -> StrategyCallable: """ :param strategy: string or function @@ -23,16 +36,16 @@ def _expand_strategy(cls, strategy): Otherwise, return the value, implicitly assuming it's a function. """ - if isinstance(strategy, string_type): - method_name = "strategy_{0}".format(strategy) - if not hasattr(cls, method_name): - raise StrategyNotFound(strategy) - return getattr(cls, method_name) + if isinstance(strategy, str): + method_name = f"strategy_{strategy}" + if hasattr(cls, method_name): + return getattr(cls, method_name) + raise StrategyNotFound(strategy) return strategy - def __call__(self, *args, **kwargs): + def __call__(self, config: deepmerge.merger.Merger, path: list, base: Any, nxt: Any) -> Any: for s in self._strategies: - ret_val = s(*args, **kwargs) + ret_val = s(config, path, base, nxt) if ret_val is not STRATEGY_END: return ret_val - raise InvalidMerge(self.NAME, args, kwargs) + raise InvalidMerge(self.NAME, config, path, base, nxt) diff --git a/deepmerge/strategy/dict.py b/deepmerge/strategy/dict.py index 8f09eb9..799518e 100644 --- a/deepmerge/strategy/dict.py +++ b/deepmerge/strategy/dict.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import deepmerge.merger from .core import StrategyList @@ -10,7 +13,7 @@ class DictStrategies(StrategyList): NAME = "dict" @staticmethod - def strategy_merge(config, path, base, nxt): + def strategy_merge(config: deepmerge.merger.Merger, path: list, base: dict, nxt: dict) -> dict: """ for keys that do not exists, use them directly. if the key exists @@ -24,7 +27,9 @@ def strategy_merge(config, path, base, nxt): return base @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override( + config: deepmerge.merger.Merger, path: list, base: dict, nxt: dict + ) -> dict: """ move all keys in nxt into base, overriding conflicts. diff --git a/deepmerge/strategy/fallback.py b/deepmerge/strategy/fallback.py index 714e8f9..7cfbaaa 100644 --- a/deepmerge/strategy/fallback.py +++ b/deepmerge/strategy/fallback.py @@ -1,6 +1,14 @@ +from __future__ import annotations + +from typing import Any, TypeVar + +import deepmerge.merger from .core import StrategyList +T = TypeVar("T") + + class FallbackStrategies(StrategyList): """ The StrategyList containing fallback strategies. @@ -9,11 +17,11 @@ class FallbackStrategies(StrategyList): NAME = "fallback" @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override(config: deepmerge.merger.Merger, path: list, base: Any, nxt: T) -> T: """use nxt, and ignore base.""" return nxt @staticmethod - def strategy_use_existing(config, path, base, nxt): + def strategy_use_existing(config: deepmerge.merger.Merger, path: list, base: T, nxt: Any) -> T: """use base, and ignore next.""" return base diff --git a/deepmerge/strategy/list.py b/deepmerge/strategy/list.py index 2e42519..dafc3a6 100644 --- a/deepmerge/strategy/list.py +++ b/deepmerge/strategy/list.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import deepmerge.merger from .core import StrategyList from ..extended_set import ExtendedSet @@ -10,22 +13,28 @@ class ListStrategies(StrategyList): NAME = "list" @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override( + config: deepmerge.merger.Merger, path: list, base: list, nxt: list + ) -> list: """use the list nxt.""" return nxt @staticmethod - def strategy_prepend(config, path, base, nxt): + def strategy_prepend( + config: deepmerge.merger.Merger, path: list, base: list, nxt: list + ) -> list: """prepend nxt to base.""" return nxt + base @staticmethod - def strategy_append(config, path, base, nxt): + def strategy_append(config: deepmerge.merger.Merger, path: list, base: list, nxt: list) -> list: """append nxt to base.""" return base + nxt @staticmethod - def strategy_append_unique(config, path, base, nxt): + def strategy_append_unique( + config: deepmerge.merger.Merger, path: list, base: list, nxt: list + ) -> list: """append items without duplicates in nxt to base.""" base_as_set = ExtendedSet(base) return base + [n for n in nxt if n not in base_as_set] diff --git a/deepmerge/strategy/set.py b/deepmerge/strategy/set.py index 55b26e3..8bea707 100644 --- a/deepmerge/strategy/set.py +++ b/deepmerge/strategy/set.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import deepmerge.merger from .core import StrategyList @@ -9,21 +12,21 @@ class SetStrategies(StrategyList): NAME = "set" @staticmethod - def strategy_union(config, path, base, nxt): + def strategy_union(config: deepmerge.merger.Merger, path: list, base: set, nxt: set) -> set: """ use all values in either base or nxt. """ return base | nxt @staticmethod - def strategy_intersect(config, path, base, nxt): + def strategy_intersect(config: deepmerge.merger.Merger, path: list, base: set, nxt: set) -> set: """ use all values in both base and nxt. """ return base & nxt @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override(config: deepmerge.merger.Merger, path: list, base: set, nxt: set) -> set: """ use the set nxt. """ diff --git a/deepmerge/strategy/type_conflict.py b/deepmerge/strategy/type_conflict.py index ebbc77a..1140368 100644 --- a/deepmerge/strategy/type_conflict.py +++ b/deepmerge/strategy/type_conflict.py @@ -1,5 +1,13 @@ +from __future__ import annotations + +from typing import Any, TypeVar + +import deepmerge.merger from .core import StrategyList +T1 = TypeVar("T1") +T2 = TypeVar("T2") + class TypeConflictStrategies(StrategyList): """contains the strategies provided for type conflicts.""" @@ -7,16 +15,20 @@ class TypeConflictStrategies(StrategyList): NAME = "type conflict" @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override(config: deepmerge.merger.Merger, path: list, base: Any, nxt: T1) -> T1: """overrides the new object over the old object""" return nxt @staticmethod - def strategy_use_existing(config, path, base, nxt): + def strategy_use_existing( + config: deepmerge.merger.Merger, path: list, base: T1, nxt: Any + ) -> T1: """uses the old object instead of the new object""" return base @staticmethod - def strategy_override_if_not_empty(config, path, base, nxt): + def strategy_override_if_not_empty( + config: deepmerge.merger.Merger, path: list, base: T1, nxt: T2 + ) -> T1 | T2: """overrides the new object over the old object only if the new object is not empty or null""" return nxt if nxt else base diff --git a/deepmerge/tests/strategy/test_core.py b/deepmerge/tests/strategy/test_core.py index afa76d7..b96922a 100644 --- a/deepmerge/tests/strategy/test_core.py +++ b/deepmerge/tests/strategy/test_core.py @@ -12,16 +12,6 @@ def always_return_custom(config, path, base, nxt): return "custom" -def test_single_value_allowed(): - """ """ - - def strat(name): - return name - - sl = StrategyList(strat) - assert sl("foo") == "foo" - - def test_first_working_strategy_is_used(): """ In the case where the StrategyList has multiple values, diff --git a/deepmerge/tests/strategy/test_type_conflict.py b/deepmerge/tests/strategy/test_type_conflict.py index a366351..ae8f576 100644 --- a/deepmerge/tests/strategy/test_type_conflict.py +++ b/deepmerge/tests/strategy/test_type_conflict.py @@ -1,6 +1,8 @@ +from typing import Dict + from deepmerge.strategy.type_conflict import TypeConflictStrategies -EMPTY_DICT = {} +EMPTY_DICT: Dict = {} CONTENT_AS_LIST = [{"key": "val"}] @@ -16,7 +18,5 @@ def test_merge_if_not_empty(): ) assert strategy == CONTENT_AS_LIST - strategy = TypeConflictStrategies.strategy_override_if_not_empty( - {}, [], CONTENT_AS_LIST, None - ) + strategy = TypeConflictStrategies.strategy_override_if_not_empty({}, [], CONTENT_AS_LIST, None) assert strategy == CONTENT_AS_LIST diff --git a/deepmerge/tests/test_full.py b/deepmerge/tests/test_full.py index 799db19..bc6bdee 100644 --- a/deepmerge/tests/test_full.py +++ b/deepmerge/tests/test_full.py @@ -31,13 +31,13 @@ def test_merge_or_raise_raises_exception(): merge_or_raise.merge(base, nxt) exc = exc_info.value assert exc.strategy_list_name == "type conflict" - assert exc.merge_args == (merge_or_raise, ["foo"], 0, "a string!") - assert exc.merge_kwargs == {} + assert exc.config == merge_or_raise + assert exc.path == ["foo"] + assert exc.base == 0 + assert exc.nxt == "a string!" -@pytest.mark.parametrize( - "base, nxt, expected", [("dooby", "fooby", "dooby"), (-10, "goo", -10)] -) +@pytest.mark.parametrize("base, nxt, expected", [("dooby", "fooby", "dooby"), (-10, "goo", -10)]) def test_use_existing(base, nxt, expected): assert conservative_merger.merge(base, nxt) == expected diff --git a/pyproject.toml b/pyproject.toml index 52553c3..edd73b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,66 @@ [build-system] -# setuptools and setuptools_scm version are explicitly set -# to allow support for python2 -requires = ["setuptools>=44", "wheel", "setuptools_scm>=5"] +requires = ["setuptools>=69", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" +[project] +name = "deepmerge" +description = "A toolset for deeply merging Python dictionaries." +authors = [ + {name = "Yusuke Tsutsumi", email = "yusuke@tsutsumi.io"}, +] + +# Dependency Information +requires-python = ">=3.8" +dependencies = [ + "typing_extensions;python_version<='3.9'", +] + +# Extra Information +readme = "README.rst" +license = {text = "MIT Licence"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Typing :: Typed", +] + +dynamic = ["version"] + +[project.urls] +Homepage = "http://deepmerge.readthedocs.io/en/latest/" +GitHub = "https://github.com/toumorokoshi/deepmerge" + +[project.optional-dependencies] +dev = [ + ## Formatting / Linting + "validate-pyproject[all]", + "pyupgrade", + "black", + "mypy", + ## Testing + "pytest", + ## Build / Release + "build", + "twine", +] + +[tool.setuptools.packages.find] +include = ["deepmerge*"] +exclude = ["tests*", "*.tests*"] + +[tool.setuptools.package-data] +deepmerge = ["py.typed"] + [tool.setuptools_scm] -write_to = "deepmerge/_version.py" \ No newline at end of file +write_to = "deepmerge/_version.py" + +[tool.black] +line-length = 100 +target-version = ["py38", "py39", "py310", "py311", "py312"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 00c5503..0000000 --- a/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[metadata] -name = deepmerge -description = a toolset to deeply merge python dictionaries. -long_description = file: README.rst -url = http://deepmerge.readthedocs.io/en/latest/ -classifiers = - Development Status :: 5 - Production/Stable - License :: OSI Approved :: MIT License - Programming Language :: Python :: 3 -python_requires = >=3.* - -[options] -zip_safe = true -packages = find: - -[options.packages.find] -exclude = - tests* - *.tests* \ No newline at end of file