From 486b9177723036c494734a3acf3f50cc3d015d2b Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Wed, 24 Jan 2024 23:35:15 +1100 Subject: [PATCH 1/7] checkpoint --- MANIFEST.in | 1 - README.rst | 19 +++-- deepmerge/__init__.py | 4 +- deepmerge/compat.py | 4 -- deepmerge/exception.py | 35 ++++++--- deepmerge/extended_set.py | 21 +++--- deepmerge/merger.py | 72 ++++++++++++------- deepmerge/py.typed | 0 deepmerge/strategy/__init__.py | 6 ++ deepmerge/strategy/core.py | 48 +++++++++---- deepmerge/strategy/dict.py | 13 +++- deepmerge/strategy/fallback.py | 16 ++++- deepmerge/strategy/list.py | 21 ++++-- deepmerge/strategy/set.py | 17 ++++- deepmerge/strategy/type_conflict.py | 20 +++++- deepmerge/tests/strategy/test_core.py | 10 --- .../tests/strategy/test_type_conflict.py | 4 +- deepmerge/tests/test_full.py | 6 +- pyproject.toml | 65 +++++++++++++++-- setup.cfg | 19 ----- 20 files changed, 284 insertions(+), 117 deletions(-) delete mode 100644 deepmerge/compat.py create mode 100644 deepmerge/py.typed delete mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in index b1fc69e..e69de29 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +0,0 @@ -include VERSION \ No newline at end of file diff --git a/README.rst b/README.rst index 24050eb..be3c01d 100644 --- a/README.rst +++ b/README.rst @@ -2,17 +2,28 @@ deepmerge ========= +.. image:: https://img.shields.io/pypi/v/deepmerge.svg + :target: https://pypi.org/project/deepmerge/ + +.. image:: https://img.shields.io/pypi/status/deepmerge.svg + :target: https://pypi.org/project/deepmerge/ + +.. image:: https://img.shields.io/pypi/pyversions/pillar.svg + :target: https://github.com/toumorokoshi/deepmerge + +.. image:: https://img.shields.io/github/license/toumorokoshi/deepmerge.svg + :target: https://github.com/toumorokoshi/deepmerge + .. image:: https://github.com/toumorokoshi/deepmerge/actions/workflows/python-package.yaml/badge.svg :target: https://github.com/toumorokoshi/deepmerge/actions/workflows/python-package.yaml -A tools to handle merging of -nested data structures in python. +A tool to handle merging of nested data structures in Python. ------------ Installation ------------ -deepmerge is available on `pypi `_: +deepmerge is available on `pypi `_: .. code-block:: bash @@ -67,4 +78,4 @@ Example You can also pass in your own merge functions, instead of a string. -For more information, see the `docs `_ \ No newline at end of file +For more information, see the `docs `_ diff --git a/deepmerge/__init__.py b/deepmerge/__init__.py index b7686cf..d6477fe 100644 --- a/deepmerge/__init__.py +++ b/deepmerge/__init__.py @@ -1,9 +1,11 @@ +from typing import List, Tuple, Type + from .merger import Merger from .strategy.core import STRATEGY_END # noqa # some standard mergers available -DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES = [ +DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES: List[Tuple[Type, str]] = [ (list, "append"), (dict, "merge"), (set, "union"), diff --git a/deepmerge/compat.py b/deepmerge/compat.py deleted file mode 100644 index d2872a6..0000000 --- a/deepmerge/compat.py +++ /dev/null @@ -1,4 +0,0 @@ -try: - string_type = basestring -except: - string_type = str diff --git a/deepmerge/exception.py b/deepmerge/exception.py index ea36dac..fd1902c 100644 --- a/deepmerge/exception.py +++ b/deepmerge/exception.py @@ -1,18 +1,35 @@ +from __future__ import annotations + +from typing import List, Any + +import deepmerge.merger + + class DeepMergeException(Exception): - pass + "Base class for all `deepmerge` Exceptions" class StrategyNotFound(DeepMergeException): - pass + "Exception for when a strategy cannot be located" class InvalidMerge(DeepMergeException): - def __init__(self, strategy_list_name, merge_args, merge_kwargs): - super(InvalidMerge, self).__init__( - "no more strategies found for {0} and arguments {1}, {2}".format( - strategy_list_name, merge_args, merge_kwargs - ) + "Exception for when unable to complete a merge operation" + + def __init__( + self, + strategy_list_name: str, + config: deepmerge.merger.Merger, + path: List, + base: Any, + nxt: Any, + ) -> None: + super().__init__( + f"Could not merge using {strategy_list_name!r} [{config=}, {path=}, {base=}, {nxt=}]" ) self.strategy_list_name = strategy_list_name - self.merge_args = merge_args - self.merge_kwargs = merge_kwargs + self.config = config + self.path = path + self.base = base + self.nxt = nxt + return diff --git a/deepmerge/extended_set.py b/deepmerge/extended_set.py index 1d51b43..3efb399 100644 --- a/deepmerge/extended_set.py +++ b/deepmerge/extended_set.py @@ -1,3 +1,6 @@ +from typing import Sequence, Any + + class ExtendedSet(set): """ ExtendedSet is an extension of set, which allows for usage @@ -9,17 +12,17 @@ class ExtendedSet(set): - unhashable types """ - def __init__(self, elements): - self._values_by_hash = {self._hash(e): e for e in elements} + def __init__(self, elements: Sequence) -> None: + self._values_by_hash = {self._hash_element(e): e for e in elements} - def _insert(self, element): - self._values_by_hash[self._hash(element)] = element + def _insert(self, element: Any) -> None: + self._values_by_hash[self._hash_element(element)] = element + return - def _hash(self, element): + def _hash_element(self, element: Any) -> int: if getattr(element, "__hash__") is not None: return hash(element) - else: - return hash(str(element)) + return hash(str(element)) - def __contains__(self, obj): - return self._hash(obj) in self._values_by_hash + def __contains__(self, obj: Any) -> bool: + return self._hash_element(obj) in self._values_by_hash diff --git a/deepmerge/merger.py b/deepmerge/merger.py index 5ccd568..4bd3455 100644 --- a/deepmerge/merger.py +++ b/deepmerge/merger.py @@ -1,44 +1,66 @@ -from .strategy.list import ListStrategies -from .strategy.dict import DictStrategies -from .strategy.set import SetStrategies -from .strategy.type_conflict import TypeConflictStrategies -from .strategy.fallback import FallbackStrategies +from __future__ import annotations +from typing import Dict, List, Type, Tuple, Any, Union, Sequence, Callable -class Merger(object): +from . import strategy as s + + +class Merger: """ + Merges objects based on provided strategies + :param type_strategies, List[Tuple]: a list of (Type, Strategy) pairs that should be used against incoming types. For example: (dict, "override"). """ - PROVIDED_TYPE_STRATEGIES = { - list: ListStrategies, - dict: DictStrategies, - set: SetStrategies, + PROVIDED_TYPE_STRATEGIES: Dict[Type, Type[s.StrategyList]] = { + list: s.ListStrategies, + dict: s.DictStrategies, + set: s.SetStrategies, } - def __init__(self, type_strategies, fallback_strategies, type_conflict_strategies): - self._fallback_strategy = FallbackStrategies(fallback_strategies) + def __init__( + self, + type_strategies: Sequence[ + Tuple[Type, Union[s.StrategyCallable, s.StrategyListInitable]] + ], + fallback_strategies: s.StrategyListInitable, + type_conflict_strategies: s.StrategyListInitable, + ) -> None: + self._fallback_strategy = s.FallbackStrategies(fallback_strategies) + self._type_conflict_strategy = s.TypeConflictStrategies( + type_conflict_strategies + ) - expanded_type_strategies = [] + self._type_strategies: List[Tuple[Type, s.StrategyCallable]] = [] for typ, strategy in type_strategies: if typ in self.PROVIDED_TYPE_STRATEGIES: - strategy = self.PROVIDED_TYPE_STRATEGIES[typ](strategy) - expanded_type_strategies.append((typ, strategy)) - self._type_strategies = expanded_type_strategies + # Customise a StrategyList instance for this type + self._type_strategies.append( + (typ, self.PROVIDED_TYPE_STRATEGIES[typ](strategy)) + ) + elif callable(strategy): + self._type_strategies.append((typ, strategy)) + else: + raise ValueError(f"Cannot handle ({typ}, {strategy})") + return - self._type_conflict_strategy = TypeConflictStrategies(type_conflict_strategies) - - def merge(self, base, nxt): + def merge(self, base: Any, nxt: Any) -> Any: return self.value_strategy([], base, nxt) - def type_conflict_strategy(self, *args): - return self._type_conflict_strategy(self, *args) + def type_conflict_strategy(self, path: List, base: Any, nxt: Any) -> Any: + return self._type_conflict_strategy(self, path, base, nxt) - def value_strategy(self, path, base, nxt): + def value_strategy(self, path: List, base: Any, nxt: Any) -> Any: + # Check for strategy based on type of base, next for typ, strategy in self._type_strategies: if isinstance(base, typ) and isinstance(nxt, typ): + # We have a strategy for this type return strategy(self, path, base, nxt) - if not (isinstance(base, type(nxt)) or isinstance(nxt, type(base))): - return self.type_conflict_strategy(path, base, nxt) - return self._fallback_strategy(self, path, base, nxt) + + if isinstance(base, type(nxt)) or isinstance(nxt, type(base)): + # no known strategy but base, next are similar types + return self._fallback_strategy(self, path, base, nxt) + + # No known strategy and base, next are different types. + return self.type_conflict_strategy(path, base, nxt) diff --git a/deepmerge/py.typed b/deepmerge/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/deepmerge/strategy/__init__.py b/deepmerge/strategy/__init__.py index e69de29..d62ea7c 100644 --- a/deepmerge/strategy/__init__.py +++ b/deepmerge/strategy/__init__.py @@ -0,0 +1,6 @@ +from .core import StrategyList, StrategyCallable, StrategyListInitable +from .dict import DictStrategies +from .fallback import FallbackStrategies +from .list import ListStrategies +from .set import SetStrategies +from .type_conflict import TypeConflictStrategies diff --git a/deepmerge/strategy/core.py b/deepmerge/strategy/core.py index 91fcfbb..2b2f3cc 100644 --- a/deepmerge/strategy/core.py +++ b/deepmerge/strategy/core.py @@ -1,20 +1,38 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, List, Callable, Any, Union + +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extentions import TypeAlias + +import deepmerge.merger from ..exception import StrategyNotFound, InvalidMerge -from ..compat import string_type STRATEGY_END = object() +StrategyCallable: TypeAlias = "Callable[[deepmerge.merger.Merger, List, Any, Any], Any]" +StrategyListInitable: TypeAlias = ( + "Union[str, StrategyCallable, List[Union[str, StrategyCallable]]]" +) -class StrategyList(object): - NAME = None +class StrategyList: + NAME: str - def __init__(self, strategy_list): + def __init__(self, strategy_list: StrategyListInitable) -> None: if not isinstance(strategy_list, list): strategy_list = [strategy_list] - self._strategies = [self._expand_strategy(s) for s in strategy_list] + self._strategies: List[StrategyCallable] = [ + self._expand_strategy(s) for s in strategy_list + ] @classmethod - def _expand_strategy(cls, strategy): + def _expand_strategy( + cls, strategy: Union[str, StrategyCallable] + ) -> StrategyCallable: """ :param strategy: string or function @@ -23,16 +41,18 @@ def _expand_strategy(cls, strategy): Otherwise, return the value, implicitly assuming it's a function. """ - if isinstance(strategy, string_type): - method_name = "strategy_{0}".format(strategy) - if not hasattr(cls, method_name): - raise StrategyNotFound(strategy) - return getattr(cls, method_name) + if isinstance(strategy, str): + method_name = f"strategy_{strategy}" + if hasattr(cls, method_name): + return getattr(cls, method_name) + raise StrategyNotFound(strategy) return strategy - def __call__(self, *args, **kwargs): + def __call__( + self, config: deepmerge.merger.Merger, path: List, base: Any, nxt: Any + ) -> Any: for s in self._strategies: - ret_val = s(*args, **kwargs) + ret_val = s(config, path, base, nxt) if ret_val is not STRATEGY_END: return ret_val - raise InvalidMerge(self.NAME, args, kwargs) + raise InvalidMerge(self.NAME, config, path, base, nxt) diff --git a/deepmerge/strategy/dict.py b/deepmerge/strategy/dict.py index 8f09eb9..b883558 100644 --- a/deepmerge/strategy/dict.py +++ b/deepmerge/strategy/dict.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from typing import Dict, List + +import deepmerge.merger from .core import StrategyList @@ -10,7 +15,9 @@ class DictStrategies(StrategyList): NAME = "dict" @staticmethod - def strategy_merge(config, path, base, nxt): + def strategy_merge( + config: deepmerge.merger.Merger, path: List, base: Dict, nxt: Dict + ) -> Dict: """ for keys that do not exists, use them directly. if the key exists @@ -24,7 +31,9 @@ def strategy_merge(config, path, base, nxt): return base @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override( + config: deepmerge.merger.Merger, path: List, base: Dict, nxt: Dict + ) -> Dict: """ move all keys in nxt into base, overriding conflicts. diff --git a/deepmerge/strategy/fallback.py b/deepmerge/strategy/fallback.py index 714e8f9..c024594 100644 --- a/deepmerge/strategy/fallback.py +++ b/deepmerge/strategy/fallback.py @@ -1,6 +1,14 @@ +from __future__ import annotations + +from typing import List, Any, TypeVar + +import deepmerge.merger from .core import StrategyList +T = TypeVar("T") + + class FallbackStrategies(StrategyList): """ The StrategyList containing fallback strategies. @@ -9,11 +17,15 @@ class FallbackStrategies(StrategyList): NAME = "fallback" @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override( + config: deepmerge.merger.Merger, path: List, base: Any, nxt: T + ) -> T: """use nxt, and ignore base.""" return nxt @staticmethod - def strategy_use_existing(config, path, base, nxt): + def strategy_use_existing( + config: deepmerge.merger.Merger, path: List, base: T, nxt: Any + ) -> T: """use base, and ignore next.""" return base diff --git a/deepmerge/strategy/list.py b/deepmerge/strategy/list.py index 2e42519..704d433 100644 --- a/deepmerge/strategy/list.py +++ b/deepmerge/strategy/list.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from typing import List + +import deepmerge.merger from .core import StrategyList from ..extended_set import ExtendedSet @@ -10,22 +15,30 @@ class ListStrategies(StrategyList): NAME = "list" @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override( + config: deepmerge.merger.Merger, path: List, base: List, nxt: List + ) -> List: """use the list nxt.""" return nxt @staticmethod - def strategy_prepend(config, path, base, nxt): + def strategy_prepend( + config: deepmerge.merger.Merger, path: List, base: List, nxt: List + ) -> List: """prepend nxt to base.""" return nxt + base @staticmethod - def strategy_append(config, path, base, nxt): + def strategy_append( + config: deepmerge.merger.Merger, path: List, base: List, nxt: List + ) -> List: """append nxt to base.""" return base + nxt @staticmethod - def strategy_append_unique(config, path, base, nxt): + def strategy_append_unique( + config: deepmerge.merger.Merger, path: List, base: List, nxt: List + ) -> List: """append items without duplicates in nxt to base.""" base_as_set = ExtendedSet(base) return base + [n for n in nxt if n not in base_as_set] diff --git a/deepmerge/strategy/set.py b/deepmerge/strategy/set.py index 55b26e3..40cf4b0 100644 --- a/deepmerge/strategy/set.py +++ b/deepmerge/strategy/set.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from typing import List, Set + +import deepmerge.merger from .core import StrategyList @@ -9,21 +14,27 @@ class SetStrategies(StrategyList): NAME = "set" @staticmethod - def strategy_union(config, path, base, nxt): + def strategy_union( + config: deepmerge.merger.Merger, path: List, base: Set, nxt: Set + ) -> Set: """ use all values in either base or nxt. """ return base | nxt @staticmethod - def strategy_intersect(config, path, base, nxt): + def strategy_intersect( + config: deepmerge.merger.Merger, path: List, base: Set, nxt: Set + ) -> Set: """ use all values in both base and nxt. """ return base & nxt @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override( + config: deepmerge.merger.Merger, path: List, base: Set, nxt: Set + ) -> Set: """ use the set nxt. """ diff --git a/deepmerge/strategy/type_conflict.py b/deepmerge/strategy/type_conflict.py index ebbc77a..6d32bfc 100644 --- a/deepmerge/strategy/type_conflict.py +++ b/deepmerge/strategy/type_conflict.py @@ -1,5 +1,13 @@ +from __future__ import annotations + +from typing import List, Union, Any, TypeVar + +import deepmerge.merger from .core import StrategyList +T1 = TypeVar("T1") +T2 = TypeVar("T2") + class TypeConflictStrategies(StrategyList): """contains the strategies provided for type conflicts.""" @@ -7,16 +15,22 @@ class TypeConflictStrategies(StrategyList): NAME = "type conflict" @staticmethod - def strategy_override(config, path, base, nxt): + def strategy_override( + config: deepmerge.merger.Merger, path: List, base: Any, nxt: T1 + ) -> T1: """overrides the new object over the old object""" return nxt @staticmethod - def strategy_use_existing(config, path, base, nxt): + def strategy_use_existing( + config: deepmerge.merger.Merger, path: List, base: T1, nxt: Any + ) -> T1: """uses the old object instead of the new object""" return base @staticmethod - def strategy_override_if_not_empty(config, path, base, nxt): + def strategy_override_if_not_empty( + config: deepmerge.merger.Merger, path: List, base: T1, nxt: T2 + ) -> Union[T1, T2]: """overrides the new object over the old object only if the new object is not empty or null""" return nxt if nxt else base diff --git a/deepmerge/tests/strategy/test_core.py b/deepmerge/tests/strategy/test_core.py index afa76d7..b96922a 100644 --- a/deepmerge/tests/strategy/test_core.py +++ b/deepmerge/tests/strategy/test_core.py @@ -12,16 +12,6 @@ def always_return_custom(config, path, base, nxt): return "custom" -def test_single_value_allowed(): - """ """ - - def strat(name): - return name - - sl = StrategyList(strat) - assert sl("foo") == "foo" - - def test_first_working_strategy_is_used(): """ In the case where the StrategyList has multiple values, diff --git a/deepmerge/tests/strategy/test_type_conflict.py b/deepmerge/tests/strategy/test_type_conflict.py index a366351..88487cc 100644 --- a/deepmerge/tests/strategy/test_type_conflict.py +++ b/deepmerge/tests/strategy/test_type_conflict.py @@ -1,6 +1,8 @@ +from typing import Dict + from deepmerge.strategy.type_conflict import TypeConflictStrategies -EMPTY_DICT = {} +EMPTY_DICT: Dict = {} CONTENT_AS_LIST = [{"key": "val"}] diff --git a/deepmerge/tests/test_full.py b/deepmerge/tests/test_full.py index 799db19..7a1fe9f 100644 --- a/deepmerge/tests/test_full.py +++ b/deepmerge/tests/test_full.py @@ -31,8 +31,10 @@ def test_merge_or_raise_raises_exception(): merge_or_raise.merge(base, nxt) exc = exc_info.value assert exc.strategy_list_name == "type conflict" - assert exc.merge_args == (merge_or_raise, ["foo"], 0, "a string!") - assert exc.merge_kwargs == {} + assert exc.config == merge_or_raise + assert exc.path == ["foo"] + assert exc.base == 0 + assert exc.nxt == "a string!" @pytest.mark.parametrize( diff --git a/pyproject.toml b/pyproject.toml index 52553c3..4467e31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,65 @@ [build-system] -# setuptools and setuptools_scm version are explicitly set -# to allow support for python2 -requires = ["setuptools>=44", "wheel", "setuptools_scm>=5"] +requires = ["setuptools>=69", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" +[project] +name = "deepmerge" +description = "A toolset for deeply merging Python dictionaries." +authors = [ + {name = "Yusuke Tsutsumi"}, +] + +# Dependency Information +requires-python = ">=3.7" +dependencies = [ + "typing_extensions;python_version<='3.9'", +] + +# Extra Information +readme = "README.rst" +license = {text = "MIT Licence"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Typing :: Typed", +] + +dynamic = ["version"] + +[project.urls] +Homepage = "http://deepmerge.readthedocs.io/en/latest/" +GitHub = "https://github.com/toumorokoshi/deepmerge" + +[project.optional-dependencies] +dev = [ + ## Formatting / Linting + "validate-pyproject[all]", + "pyupgrade", + "black", + "pylint", + "mypy", + ## Testing + "pytest", +] + +[tool.setuptools.packages.find] +include = ["deepmerge*"] +exclude = ["tests*", "*.tests*"] + +[tool.setuptools.package-data] +deepmerge = ["py.typed"] + [tool.setuptools_scm] -write_to = "deepmerge/_version.py" \ No newline at end of file +write_to = "deepmerge/_version.py" + +[tool.black] +line-length = 88 +target-version = ["py37", "py38", "py39", "py310", "py311", "py312"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 00c5503..0000000 --- a/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[metadata] -name = deepmerge -description = a toolset to deeply merge python dictionaries. -long_description = file: README.rst -url = http://deepmerge.readthedocs.io/en/latest/ -classifiers = - Development Status :: 5 - Production/Stable - License :: OSI Approved :: MIT License - Programming Language :: Python :: 3 -python_requires = >=3.* - -[options] -zip_safe = true -packages = find: - -[options.packages.find] -exclude = - tests* - *.tests* \ No newline at end of file From 4573b2d8292f034333384fd858fdd3958b14e68e Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 26 Jan 2024 19:36:59 +1100 Subject: [PATCH 2/7] Drop python 3.7 support, run pyupgrade, black --- README.rst | 11 ++++++++++ deepmerge/__init__.py | 4 +--- deepmerge/exception.py | 2 +- deepmerge/merger.py | 20 +++++++------------ deepmerge/strategy/core.py | 16 ++++----------- deepmerge/strategy/dict.py | 8 +++----- deepmerge/strategy/fallback.py | 8 ++------ deepmerge/strategy/list.py | 16 +++++++-------- deepmerge/strategy/set.py | 12 +++-------- deepmerge/strategy/type_conflict.py | 10 ++++------ .../tests/strategy/test_type_conflict.py | 4 +--- deepmerge/tests/test_full.py | 4 +--- pyproject.toml | 9 ++++----- 13 files changed, 49 insertions(+), 75 deletions(-) diff --git a/README.rst b/README.rst index be3c01d..c8b61c5 100644 --- a/README.rst +++ b/README.rst @@ -79,3 +79,14 @@ Example You can also pass in your own merge functions, instead of a string. For more information, see the `docs `_ + +------------------ +Supported Versions +------------------ + +deepmerge is supported on Python 3.8+. + +For older Python versions the last supported version of deepmerge is listed +below: + +- 3.7 : 1.1.1 diff --git a/deepmerge/__init__.py b/deepmerge/__init__.py index d6477fe..15a2757 100644 --- a/deepmerge/__init__.py +++ b/deepmerge/__init__.py @@ -15,9 +15,7 @@ # in the case of type mismatches, # the value from the second object # will override the previous one. -always_merger = Merger( - DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["override"], ["override"] -) +always_merger = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["override"], ["override"]) # this merge strategies attempts # to merge (append for list, unify for dicts) diff --git a/deepmerge/exception.py b/deepmerge/exception.py index fd1902c..dd7584a 100644 --- a/deepmerge/exception.py +++ b/deepmerge/exception.py @@ -20,7 +20,7 @@ def __init__( self, strategy_list_name: str, config: deepmerge.merger.Merger, - path: List, + path: list, base: Any, nxt: Any, ) -> None: diff --git a/deepmerge/merger.py b/deepmerge/merger.py index 4bd3455..2349e77 100644 --- a/deepmerge/merger.py +++ b/deepmerge/merger.py @@ -13,7 +13,7 @@ class Merger: that should be used against incoming types. For example: (dict, "override"). """ - PROVIDED_TYPE_STRATEGIES: Dict[Type, Type[s.StrategyList]] = { + PROVIDED_TYPE_STRATEGIES: dict[type, type[s.StrategyList]] = { list: s.ListStrategies, dict: s.DictStrategies, set: s.SetStrategies, @@ -21,24 +21,18 @@ class Merger: def __init__( self, - type_strategies: Sequence[ - Tuple[Type, Union[s.StrategyCallable, s.StrategyListInitable]] - ], + type_strategies: Sequence[tuple[type, s.StrategyCallable | s.StrategyListInitable]], fallback_strategies: s.StrategyListInitable, type_conflict_strategies: s.StrategyListInitable, ) -> None: self._fallback_strategy = s.FallbackStrategies(fallback_strategies) - self._type_conflict_strategy = s.TypeConflictStrategies( - type_conflict_strategies - ) + self._type_conflict_strategy = s.TypeConflictStrategies(type_conflict_strategies) - self._type_strategies: List[Tuple[Type, s.StrategyCallable]] = [] + self._type_strategies: list[tuple[type, s.StrategyCallable]] = [] for typ, strategy in type_strategies: if typ in self.PROVIDED_TYPE_STRATEGIES: # Customise a StrategyList instance for this type - self._type_strategies.append( - (typ, self.PROVIDED_TYPE_STRATEGIES[typ](strategy)) - ) + self._type_strategies.append((typ, self.PROVIDED_TYPE_STRATEGIES[typ](strategy))) elif callable(strategy): self._type_strategies.append((typ, strategy)) else: @@ -48,10 +42,10 @@ def __init__( def merge(self, base: Any, nxt: Any) -> Any: return self.value_strategy([], base, nxt) - def type_conflict_strategy(self, path: List, base: Any, nxt: Any) -> Any: + def type_conflict_strategy(self, path: list, base: Any, nxt: Any) -> Any: return self._type_conflict_strategy(self, path, base, nxt) - def value_strategy(self, path: List, base: Any, nxt: Any) -> Any: + def value_strategy(self, path: list, base: Any, nxt: Any) -> Any: # Check for strategy based on type of base, next for typ, strategy in self._type_strategies: if isinstance(base, typ) and isinstance(nxt, typ): diff --git a/deepmerge/strategy/core.py b/deepmerge/strategy/core.py index 2b2f3cc..1c39735 100644 --- a/deepmerge/strategy/core.py +++ b/deepmerge/strategy/core.py @@ -14,9 +14,7 @@ STRATEGY_END = object() StrategyCallable: TypeAlias = "Callable[[deepmerge.merger.Merger, List, Any, Any], Any]" -StrategyListInitable: TypeAlias = ( - "Union[str, StrategyCallable, List[Union[str, StrategyCallable]]]" -) +StrategyListInitable: TypeAlias = "Union[str, StrategyCallable, List[Union[str, StrategyCallable]]]" class StrategyList: @@ -25,14 +23,10 @@ class StrategyList: def __init__(self, strategy_list: StrategyListInitable) -> None: if not isinstance(strategy_list, list): strategy_list = [strategy_list] - self._strategies: List[StrategyCallable] = [ - self._expand_strategy(s) for s in strategy_list - ] + self._strategies: list[StrategyCallable] = [self._expand_strategy(s) for s in strategy_list] @classmethod - def _expand_strategy( - cls, strategy: Union[str, StrategyCallable] - ) -> StrategyCallable: + def _expand_strategy(cls, strategy: str | StrategyCallable) -> StrategyCallable: """ :param strategy: string or function @@ -48,9 +42,7 @@ def _expand_strategy( raise StrategyNotFound(strategy) return strategy - def __call__( - self, config: deepmerge.merger.Merger, path: List, base: Any, nxt: Any - ) -> Any: + def __call__(self, config: deepmerge.merger.Merger, path: list, base: Any, nxt: Any) -> Any: for s in self._strategies: ret_val = s(config, path, base, nxt) if ret_val is not STRATEGY_END: diff --git a/deepmerge/strategy/dict.py b/deepmerge/strategy/dict.py index b883558..bd9a62f 100644 --- a/deepmerge/strategy/dict.py +++ b/deepmerge/strategy/dict.py @@ -15,9 +15,7 @@ class DictStrategies(StrategyList): NAME = "dict" @staticmethod - def strategy_merge( - config: deepmerge.merger.Merger, path: List, base: Dict, nxt: Dict - ) -> Dict: + def strategy_merge(config: deepmerge.merger.Merger, path: list, base: dict, nxt: dict) -> dict: """ for keys that do not exists, use them directly. if the key exists @@ -32,8 +30,8 @@ def strategy_merge( @staticmethod def strategy_override( - config: deepmerge.merger.Merger, path: List, base: Dict, nxt: Dict - ) -> Dict: + config: deepmerge.merger.Merger, path: list, base: dict, nxt: dict + ) -> dict: """ move all keys in nxt into base, overriding conflicts. diff --git a/deepmerge/strategy/fallback.py b/deepmerge/strategy/fallback.py index c024594..1be327c 100644 --- a/deepmerge/strategy/fallback.py +++ b/deepmerge/strategy/fallback.py @@ -17,15 +17,11 @@ class FallbackStrategies(StrategyList): NAME = "fallback" @staticmethod - def strategy_override( - config: deepmerge.merger.Merger, path: List, base: Any, nxt: T - ) -> T: + def strategy_override(config: deepmerge.merger.Merger, path: list, base: Any, nxt: T) -> T: """use nxt, and ignore base.""" return nxt @staticmethod - def strategy_use_existing( - config: deepmerge.merger.Merger, path: List, base: T, nxt: Any - ) -> T: + def strategy_use_existing(config: deepmerge.merger.Merger, path: list, base: T, nxt: Any) -> T: """use base, and ignore next.""" return base diff --git a/deepmerge/strategy/list.py b/deepmerge/strategy/list.py index 704d433..26b77fa 100644 --- a/deepmerge/strategy/list.py +++ b/deepmerge/strategy/list.py @@ -16,29 +16,27 @@ class ListStrategies(StrategyList): @staticmethod def strategy_override( - config: deepmerge.merger.Merger, path: List, base: List, nxt: List - ) -> List: + config: deepmerge.merger.Merger, path: list, base: list, nxt: list + ) -> list: """use the list nxt.""" return nxt @staticmethod def strategy_prepend( - config: deepmerge.merger.Merger, path: List, base: List, nxt: List - ) -> List: + config: deepmerge.merger.Merger, path: list, base: list, nxt: list + ) -> list: """prepend nxt to base.""" return nxt + base @staticmethod - def strategy_append( - config: deepmerge.merger.Merger, path: List, base: List, nxt: List - ) -> List: + def strategy_append(config: deepmerge.merger.Merger, path: list, base: list, nxt: list) -> list: """append nxt to base.""" return base + nxt @staticmethod def strategy_append_unique( - config: deepmerge.merger.Merger, path: List, base: List, nxt: List - ) -> List: + config: deepmerge.merger.Merger, path: list, base: list, nxt: list + ) -> list: """append items without duplicates in nxt to base.""" base_as_set = ExtendedSet(base) return base + [n for n in nxt if n not in base_as_set] diff --git a/deepmerge/strategy/set.py b/deepmerge/strategy/set.py index 40cf4b0..3b2587d 100644 --- a/deepmerge/strategy/set.py +++ b/deepmerge/strategy/set.py @@ -14,27 +14,21 @@ class SetStrategies(StrategyList): NAME = "set" @staticmethod - def strategy_union( - config: deepmerge.merger.Merger, path: List, base: Set, nxt: Set - ) -> Set: + def strategy_union(config: deepmerge.merger.Merger, path: list, base: set, nxt: set) -> set: """ use all values in either base or nxt. """ return base | nxt @staticmethod - def strategy_intersect( - config: deepmerge.merger.Merger, path: List, base: Set, nxt: Set - ) -> Set: + def strategy_intersect(config: deepmerge.merger.Merger, path: list, base: set, nxt: set) -> set: """ use all values in both base and nxt. """ return base & nxt @staticmethod - def strategy_override( - config: deepmerge.merger.Merger, path: List, base: Set, nxt: Set - ) -> Set: + def strategy_override(config: deepmerge.merger.Merger, path: list, base: set, nxt: set) -> set: """ use the set nxt. """ diff --git a/deepmerge/strategy/type_conflict.py b/deepmerge/strategy/type_conflict.py index 6d32bfc..5d579e6 100644 --- a/deepmerge/strategy/type_conflict.py +++ b/deepmerge/strategy/type_conflict.py @@ -15,22 +15,20 @@ class TypeConflictStrategies(StrategyList): NAME = "type conflict" @staticmethod - def strategy_override( - config: deepmerge.merger.Merger, path: List, base: Any, nxt: T1 - ) -> T1: + def strategy_override(config: deepmerge.merger.Merger, path: list, base: Any, nxt: T1) -> T1: """overrides the new object over the old object""" return nxt @staticmethod def strategy_use_existing( - config: deepmerge.merger.Merger, path: List, base: T1, nxt: Any + config: deepmerge.merger.Merger, path: list, base: T1, nxt: Any ) -> T1: """uses the old object instead of the new object""" return base @staticmethod def strategy_override_if_not_empty( - config: deepmerge.merger.Merger, path: List, base: T1, nxt: T2 - ) -> Union[T1, T2]: + config: deepmerge.merger.Merger, path: list, base: T1, nxt: T2 + ) -> T1 | T2: """overrides the new object over the old object only if the new object is not empty or null""" return nxt if nxt else base diff --git a/deepmerge/tests/strategy/test_type_conflict.py b/deepmerge/tests/strategy/test_type_conflict.py index 88487cc..ae8f576 100644 --- a/deepmerge/tests/strategy/test_type_conflict.py +++ b/deepmerge/tests/strategy/test_type_conflict.py @@ -18,7 +18,5 @@ def test_merge_if_not_empty(): ) assert strategy == CONTENT_AS_LIST - strategy = TypeConflictStrategies.strategy_override_if_not_empty( - {}, [], CONTENT_AS_LIST, None - ) + strategy = TypeConflictStrategies.strategy_override_if_not_empty({}, [], CONTENT_AS_LIST, None) assert strategy == CONTENT_AS_LIST diff --git a/deepmerge/tests/test_full.py b/deepmerge/tests/test_full.py index 7a1fe9f..bc6bdee 100644 --- a/deepmerge/tests/test_full.py +++ b/deepmerge/tests/test_full.py @@ -37,9 +37,7 @@ def test_merge_or_raise_raises_exception(): assert exc.nxt == "a string!" -@pytest.mark.parametrize( - "base, nxt, expected", [("dooby", "fooby", "dooby"), (-10, "goo", -10)] -) +@pytest.mark.parametrize("base, nxt, expected", [("dooby", "fooby", "dooby"), (-10, "goo", -10)]) def test_use_existing(base, nxt, expected): assert conservative_merger.merge(base, nxt) == expected diff --git a/pyproject.toml b/pyproject.toml index 4467e31..2fc99f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,11 @@ build-backend = "setuptools.build_meta" name = "deepmerge" description = "A toolset for deeply merging Python dictionaries." authors = [ - {name = "Yusuke Tsutsumi"}, + {name = "Yusuke Tsutsumi", email = "yusuke@tsutsumi.io"}, ] # Dependency Information -requires-python = ">=3.7" +requires-python = ">=3.8" dependencies = [ "typing_extensions;python_version<='3.9'", ] @@ -21,7 +21,6 @@ license = {text = "MIT Licence"} classifiers = [ "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -61,5 +60,5 @@ deepmerge = ["py.typed"] write_to = "deepmerge/_version.py" [tool.black] -line-length = 88 -target-version = ["py37", "py38", "py39", "py310", "py311", "py312"] +line-length = 100 +target-version = ["py38", "py39", "py310", "py311", "py312"] From 766e156b03c193c6f976dea1b955d9c40e5ea8d7 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 26 Jan 2024 19:39:18 +1100 Subject: [PATCH 3/7] Remove setup.cfg from Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 75ba49c..4f87f8d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .venv: pyproject.toml python -m virtualenv .venv -.venv/deps: .venv pyproject.toml setup.cfg +.venv/deps: .venv pyproject.toml .venv/bin/python -m pip install . build pytest twine touch .venv/deps @@ -17,4 +17,4 @@ lint: .venv/deps test: .venv/deps .venv/bin/python -m pytest deepmerge -ready-pr: test lint \ No newline at end of file +ready-pr: test lint From d3e7fb4a34d35fe1b67b9d48c8b699b0025058ef Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 26 Jan 2024 19:54:01 +1100 Subject: [PATCH 4/7] Remove unneeded imports --- deepmerge/__init__.py | 10 +++++----- deepmerge/exception.py | 2 +- deepmerge/merger.py | 2 +- deepmerge/strategy/core.py | 7 ++++--- deepmerge/strategy/dict.py | 2 -- deepmerge/strategy/fallback.py | 2 +- deepmerge/strategy/list.py | 2 -- deepmerge/strategy/set.py | 2 -- deepmerge/strategy/type_conflict.py | 2 +- 9 files changed, 13 insertions(+), 18 deletions(-) diff --git a/deepmerge/__init__.py b/deepmerge/__init__.py index 15a2757..503089c 100644 --- a/deepmerge/__init__.py +++ b/deepmerge/__init__.py @@ -1,11 +1,11 @@ -from typing import List, Tuple, Type +from __future__ import annotations from .merger import Merger from .strategy.core import STRATEGY_END # noqa # some standard mergers available -DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES: List[Tuple[Type, str]] = [ +DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES: list[tuple[type, str]] = [ (list, "append"), (dict, "merge"), (set, "union"), @@ -15,13 +15,13 @@ # in the case of type mismatches, # the value from the second object # will override the previous one. -always_merger = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["override"], ["override"]) +always_merger: Merger = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["override"], ["override"]) # this merge strategies attempts # to merge (append for list, unify for dicts) # if possible, but raises an exception # in the case of type conflicts. -merge_or_raise = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, [], []) +merge_or_raise: Merger = Merger(DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, [], []) # a conservative merge tactic: # for data structures with a specific @@ -29,6 +29,6 @@ # similar to always_merger but instead # keeps existing values when faced # with a type conflict. -conservative_merger = Merger( +conservative_merger: Merger = Merger( DEFAULT_TYPE_SPECIFIC_MERGE_STRATEGIES, ["use_existing"], ["use_existing"] ) diff --git a/deepmerge/exception.py b/deepmerge/exception.py index dd7584a..c266c09 100644 --- a/deepmerge/exception.py +++ b/deepmerge/exception.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List, Any +from typing import Any import deepmerge.merger diff --git a/deepmerge/merger.py b/deepmerge/merger.py index 2349e77..6c85ff9 100644 --- a/deepmerge/merger.py +++ b/deepmerge/merger.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, List, Type, Tuple, Any, Union, Sequence, Callable +from typing import Any, Sequence, Callable from . import strategy as s diff --git a/deepmerge/strategy/core.py b/deepmerge/strategy/core.py index 1c39735..b91255d 100644 --- a/deepmerge/strategy/core.py +++ b/deepmerge/strategy/core.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING, List, Callable, Any, Union +from typing import Callable, Any if sys.version_info >= (3, 10): from typing import TypeAlias @@ -13,8 +13,9 @@ STRATEGY_END = object() -StrategyCallable: TypeAlias = "Callable[[deepmerge.merger.Merger, List, Any, Any], Any]" -StrategyListInitable: TypeAlias = "Union[str, StrategyCallable, List[Union[str, StrategyCallable]]]" +# Note: We use string annotations here to prevent circular import caused by Merger +StrategyCallable: TypeAlias = "Callable[[deepmerge.merger.Merger, list, Any, Any], Any]" +StrategyListInitable: TypeAlias = "str | StrategyCallable | list[str | StrategyCallable]" class StrategyList: diff --git a/deepmerge/strategy/dict.py b/deepmerge/strategy/dict.py index bd9a62f..799518e 100644 --- a/deepmerge/strategy/dict.py +++ b/deepmerge/strategy/dict.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Dict, List - import deepmerge.merger from .core import StrategyList diff --git a/deepmerge/strategy/fallback.py b/deepmerge/strategy/fallback.py index 1be327c..7cfbaaa 100644 --- a/deepmerge/strategy/fallback.py +++ b/deepmerge/strategy/fallback.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List, Any, TypeVar +from typing import Any, TypeVar import deepmerge.merger from .core import StrategyList diff --git a/deepmerge/strategy/list.py b/deepmerge/strategy/list.py index 26b77fa..dafc3a6 100644 --- a/deepmerge/strategy/list.py +++ b/deepmerge/strategy/list.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import List - import deepmerge.merger from .core import StrategyList from ..extended_set import ExtendedSet diff --git a/deepmerge/strategy/set.py b/deepmerge/strategy/set.py index 3b2587d..8bea707 100644 --- a/deepmerge/strategy/set.py +++ b/deepmerge/strategy/set.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import List, Set - import deepmerge.merger from .core import StrategyList diff --git a/deepmerge/strategy/type_conflict.py b/deepmerge/strategy/type_conflict.py index 5d579e6..1140368 100644 --- a/deepmerge/strategy/type_conflict.py +++ b/deepmerge/strategy/type_conflict.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List, Union, Any, TypeVar +from typing import Any, TypeVar import deepmerge.merger from .core import StrategyList From 739e3c0adef77938ac761eb80f957e77ea291dc1 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 26 Jan 2024 20:14:29 +1100 Subject: [PATCH 5/7] Move all dev deps to pyproject.toml, make .venv more portable --- Makefile | 13 +++++++------ pyproject.toml | 4 +++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 4f87f8d..4013630 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,21 @@ .venv: pyproject.toml - python -m virtualenv .venv + python3 -m venv .venv .venv/deps: .venv pyproject.toml - .venv/bin/python -m pip install . build pytest twine + .venv/bin/python -m pip install .[dev] touch .venv/deps build: .venv/deps rm -rf ./dist/ - .venv/bin/python -m build . + .venv/bin/build . # only works with python 3+ lint: .venv/deps - .venv/bin/python -m pip install black==22.3.0 - .venv/bin/python -m black --check . + .venv/bin/validate-pyproject pyproject.toml + .venv/bin/black --check deepmerge + .venv/bin/mypy deepmerge test: .venv/deps - .venv/bin/python -m pytest deepmerge + .venv/bin/pytest deepmerge ready-pr: test lint diff --git a/pyproject.toml b/pyproject.toml index 2fc99f0..edd73b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,10 +43,12 @@ dev = [ "validate-pyproject[all]", "pyupgrade", "black", - "pylint", "mypy", ## Testing "pytest", + ## Build / Release + "build", + "twine", ] [tool.setuptools.packages.find] From 8c771c887c792ba3cdac6f2f37afc9be2628da96 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Jan 2024 19:33:11 +1100 Subject: [PATCH 6/7] Fix build command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4013630..806ba7f 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ build: .venv/deps rm -rf ./dist/ - .venv/bin/build . + .venv/bin/python -m build # only works with python 3+ lint: .venv/deps From 8eef6295e21482c05ff51e40baed6d344a2161b4 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 28 Jan 2024 18:37:34 +1100 Subject: [PATCH 7/7] Fix import typo --- deepmerge/strategy/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmerge/strategy/core.py b/deepmerge/strategy/core.py index b91255d..b035f57 100644 --- a/deepmerge/strategy/core.py +++ b/deepmerge/strategy/core.py @@ -6,7 +6,7 @@ if sys.version_info >= (3, 10): from typing import TypeAlias else: - from typing_extentions import TypeAlias + from typing_extensions import TypeAlias import deepmerge.merger from ..exception import StrategyNotFound, InvalidMerge