From 472d739eb52964df17f0f73212a9169eb5202aa1 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Wed, 16 Nov 2022 21:16:31 +0100 Subject: [PATCH 1/7] Remove constraint on packages --- openfisca_tasks/install.mk | 6 +++--- openfisca_tasks/lint.mk | 12 ++++++------ openfisca_tasks/publish.mk | 6 +++--- openfisca_tasks/test_code.mk | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/openfisca_tasks/install.mk b/openfisca_tasks/install.mk index d4ffd8667a..6725577621 100644 --- a/openfisca_tasks/install.mk +++ b/openfisca_tasks/install.mk @@ -1,17 +1,17 @@ ## Uninstall project's dependencies. uninstall: @$(call print_help,$@:) - @pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs pip uninstall -y + @python -m pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs python -m pip uninstall -y ## Install project's overall dependencies install-deps: @$(call print_help,$@:) - @pip install --upgrade pip twine wheel + @python -m pip install --upgrade pip twine wheel ## Install project's development dependencies. install-edit: @$(call print_help,$@:) - @pip install --upgrade --editable ".[dev]" + @python -m pip install --upgrade --editable ".[dev]" ## Delete builds and compiled python files. clean: diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 115c6267bb..5e1a6f3d73 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -11,7 +11,7 @@ check-syntax-errors: . ## Run linters to check for syntax and style errors. check-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @flake8 $? + @python -m flake8 $? @$(call print_pass,$@:) ## Run linters to check for syntax and style errors in the doc. @@ -29,14 +29,14 @@ lint-doc-%: @## able to integrate documentation improvements progresively. @## @$(call print_help,$(subst $*,%,$@:)) - @flake8 --select=D101,D102,D103,DAR openfisca_core/$* - @pylint openfisca_core/$* + @python -m flake8 --select=D101,D102,D103,DAR openfisca_core/$* + @python -m pylint openfisca_core/$* @$(call print_pass,$@:) ## Run static type checkers for type errors. check-types: @$(call print_help,$@:) - @mypy --package openfisca_core --package openfisca_web_api + @python -m mypy --package openfisca_core --package openfisca_web_api @$(call print_pass,$@:) ## Run static type checkers for type errors (strict). @@ -48,7 +48,7 @@ lint-typing-strict: \ ## Run static type checkers for type errors (strict). lint-typing-strict-%: @$(call print_help,$(subst $*,%,$@:)) - @mypy \ + @python -m mypy \ --cache-dir .mypy_cache-openfisca_core.$* \ --implicit-reexport \ --strict \ @@ -58,5 +58,5 @@ lint-typing-strict-%: ## Run code formatters to correct style errors. format-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @autopep8 $? + @python -m autopep8 $? @$(call print_pass,$@:) diff --git a/openfisca_tasks/publish.mk b/openfisca_tasks/publish.mk index 9bac9ae365..981bffa86b 100644 --- a/openfisca_tasks/publish.mk +++ b/openfisca_tasks/publish.mk @@ -5,8 +5,8 @@ build: @## This allows us to be sure tests are run against the packaged version @## of openfisca-core, the same we put in the hands of users and reusers. @$(call print_help,$@:) - @pip install --upgrade build + @python -m pip install --upgrade build @python -m build - @pip uninstall --yes openfisca-core - @find dist -name "*.whl" -exec pip install {}[dev] \; + @python -m pip uninstall --yes openfisca-core + @find dist -name "*.whl" -exec python -m pip install {}[dev] \; @$(call print_pass,$@:) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 63fdd4386a..7df7a905a5 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -10,8 +10,8 @@ install: install-deps install-edit install-test ## Enable regression testing with template repositories. install-test: @$(call print_help,$@:) - @pip install --upgrade --no-dependencies openfisca-country-template - @pip install --upgrade --no-dependencies openfisca-extension-template + @python -m pip install --upgrade --no-deps openfisca-country-template + @python -m pip install --upgrade --no-deps openfisca-extension-template ## Run openfisca-core & country/extension template tests. test-code: test-core test-country test-extension @@ -31,12 +31,12 @@ test-code: test-core test-country test-extension ## Run openfisca-core tests. test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 -d ":") @$(call print_help,$@:) - @pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ + @python -m pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ openfisca_core/commons \ openfisca_core/holders \ openfisca_core/types @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - coverage run -m \ + python -m coverage run -m \ ${openfisca} test $? \ ${openfisca_args} @$(call print_pass,$@:) @@ -63,4 +63,4 @@ test-extension: ## Print the coverage report. test-cov: @$(call print_help,$@:) - @coverage report + @python -m coverage report From f8939ca2c03a92ce443afc6ad5b3f05e893f230e Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 17 Nov 2022 19:47:03 +0100 Subject: [PATCH 2/7] Update doc & make --- openfisca_tasks/install.mk | 12 +++++++++++- openfisca_tasks/lint.mk | 12 ++++++------ openfisca_tasks/publish.mk | 6 +++--- openfisca_tasks/test_code.mk | 5 +++-- setup.py | 4 ++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/openfisca_tasks/install.mk b/openfisca_tasks/install.mk index 6725577621..d047a6eb15 100644 --- a/openfisca_tasks/install.mk +++ b/openfisca_tasks/install.mk @@ -1,7 +1,17 @@ ## Uninstall project's dependencies. uninstall: @$(call print_help,$@:) - @python -m pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs python -m pip uninstall -y + @pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs pip uninstall -y + +## Install project's overall dependencies +install-deps: + @$(call print_help,$@:) + @pip install --upgrade pip twine wheel + +## Install project's development dependencies. +install-edit: + @$(call print_help,$@:) + @pip install --upgrade --editable ".[dev]" ## Install project's overall dependencies install-deps: diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 5e1a6f3d73..115c6267bb 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -11,7 +11,7 @@ check-syntax-errors: . ## Run linters to check for syntax and style errors. check-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @python -m flake8 $? + @flake8 $? @$(call print_pass,$@:) ## Run linters to check for syntax and style errors in the doc. @@ -29,14 +29,14 @@ lint-doc-%: @## able to integrate documentation improvements progresively. @## @$(call print_help,$(subst $*,%,$@:)) - @python -m flake8 --select=D101,D102,D103,DAR openfisca_core/$* - @python -m pylint openfisca_core/$* + @flake8 --select=D101,D102,D103,DAR openfisca_core/$* + @pylint openfisca_core/$* @$(call print_pass,$@:) ## Run static type checkers for type errors. check-types: @$(call print_help,$@:) - @python -m mypy --package openfisca_core --package openfisca_web_api + @mypy --package openfisca_core --package openfisca_web_api @$(call print_pass,$@:) ## Run static type checkers for type errors (strict). @@ -48,7 +48,7 @@ lint-typing-strict: \ ## Run static type checkers for type errors (strict). lint-typing-strict-%: @$(call print_help,$(subst $*,%,$@:)) - @python -m mypy \ + @mypy \ --cache-dir .mypy_cache-openfisca_core.$* \ --implicit-reexport \ --strict \ @@ -58,5 +58,5 @@ lint-typing-strict-%: ## Run code formatters to correct style errors. format-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @python -m autopep8 $? + @autopep8 $? @$(call print_pass,$@:) diff --git a/openfisca_tasks/publish.mk b/openfisca_tasks/publish.mk index 981bffa86b..9bac9ae365 100644 --- a/openfisca_tasks/publish.mk +++ b/openfisca_tasks/publish.mk @@ -5,8 +5,8 @@ build: @## This allows us to be sure tests are run against the packaged version @## of openfisca-core, the same we put in the hands of users and reusers. @$(call print_help,$@:) - @python -m pip install --upgrade build + @pip install --upgrade build @python -m build - @python -m pip uninstall --yes openfisca-core - @find dist -name "*.whl" -exec python -m pip install {}[dev] \; + @pip uninstall --yes openfisca-core + @find dist -name "*.whl" -exec pip install {}[dev] \; @$(call print_pass,$@:) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 7df7a905a5..703cf4ec20 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -4,6 +4,7 @@ openfisca = openfisca_core.scripts.openfisca_command ## The path to the installed packages. python_packages = $(shell python -c "import sysconfig; print(sysconfig.get_paths()[\"purelib\"])") +<<<<<<< HEAD ## Run all tasks required for testing. install: install-deps install-edit install-test @@ -36,7 +37,7 @@ test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 openfisca_core/holders \ openfisca_core/types @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - python -m coverage run -m \ + coverage run -m \ ${openfisca} test $? \ ${openfisca_args} @$(call print_pass,$@:) @@ -63,4 +64,4 @@ test-extension: ## Print the coverage report. test-cov: @$(call print_help,$@:) - @python -m coverage report + @coverage report diff --git a/setup.py b/setup.py index 0eb2fa5624..504a5513ea 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ """Package config file. -This file contains all package's metadata, including the current version and -its third-party dependencies. +This file contains all the package's metadata, including the current version +and its third-party dependencies. Note: For integration testing, OpenFisca-Core relies on two other packages, From fc947b545cd39e12dfd418117ed17cbbf0ce1fc8 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Mon, 21 Nov 2022 22:35:41 +0100 Subject: [PATCH 3/7] Add nox --- README.md | 13 +++++++ noxfile.py | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 noxfile.py diff --git a/README.md b/README.md index a3f18555f1..93597f7e1b 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,19 @@ To run a single test: pytest tests/core/test_parameters.py -k test_parameter_for_period ``` +### Testing with Nox + +In order to test several Python and NumPy versions locally, we recommend you to +install [pipx](https://pypa.github.io/pipx/installation/) and configure it with +your a Python version manager like [pyenv](https://github.com/pyenv/pyenv) and +[nox](https://nox.thea.codes/en/stable/tutorial.html#installation). + +Then run: + +```sh +nox -s +``` + ## Types This repository relies on MyPy for optional dynamic & static type checking. diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000..60625ff135 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,110 @@ +"""Nox config file.""" + +import nox + +nox.options.reuse_existing_virtualenvs = True +nox.options.stop_on_first_error = True + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "style")) +@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +def style(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "check-style", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "docs")) +@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +def docs(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "lint-doc", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "mypy")) +@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +def mypy(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "check-types", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "mypy-hxc")) +@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +def mypy_hxc(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "lint-typing-strict", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-core")) +@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +def test_core(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.install("--no-deps", "openfisca-country-template") + session.install("--no-deps", "openfisca-extension-template") + session.run("make", "test-core", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-country")) +@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +def test_country(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.install("--no-deps", "openfisca-country-template") + session.install("--no-deps", "openfisca-extension-template") + session.run("make", "test-country", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-extension")) +@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +def test_extension(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.install("--no-deps", "openfisca-country-template") + session.install("--no-deps", "openfisca-extension-template") + session.run("make", "test-extension", external = True) From 59adf63c595a7dbe744a4db74e0c48ac3063273c Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 22 Nov 2022 01:52:24 +0100 Subject: [PATCH 4/7] Fix deprecated numpy signature --- .gitignore | 2 ++ openfisca_core/tracers/computation_log.py | 2 +- setup.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 00d9f19fb6..f89f5f76c2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,11 @@ *.pyc *~ .coverage +.mypy.ini .mypy_cache .mypy_cache* .noseids +.nox .project .pydevproject .pytest_cache diff --git a/openfisca_core/tracers/computation_log.py b/openfisca_core/tracers/computation_log.py index 9bb830b8d2..8f1b297dfe 100644 --- a/openfisca_core/tracers/computation_log.py +++ b/openfisca_core/tracers/computation_log.py @@ -28,7 +28,7 @@ def display( if isinstance(value, EnumArray): value = value.decode_to_str() - return numpy.array2string(value, max_line_width = float("inf")) + return numpy.array2string(value, max_line_width = None) def lines( self, diff --git a/setup.py b/setup.py index 504a5513ea..1ff4a54c36 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'dpath >= 1.5.0, < 3.0.0', 'nptyping == 1.4.4', 'numexpr >= 2.7.0, <= 3.0', - 'numpy >= 1.11, < 1.21', + 'numpy >= 1.20.3, < 1.23.0', 'psutil >= 5.4.7, < 6.0.0', 'pytest >= 4.4.1, < 6.0.0', # For openfisca test 'PyYAML >= 3.10', From 1720f67de6f9134fd4accf743044109c645c2c3e Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 22 Nov 2022 02:44:08 +0100 Subject: [PATCH 5/7] Remove nptyping --- openfisca_core/commons/formulas.py | 33 ++++++------ openfisca_core/tracers/computation_log.py | 1 + openfisca_core/types/__init__.py | 12 +---- openfisca_core/types/_data.py | 51 ------------------- .../types/{_domain.py => _types.py} | 0 setup.py | 1 - 6 files changed, 17 insertions(+), 81 deletions(-) delete mode 100644 openfisca_core/types/_data.py rename openfisca_core/types/{_domain.py => _types.py} (100%) diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index 4e13b925fb..188f6f2062 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,17 +1,13 @@ -from typing import Any, Dict, Sequence, TypeVar +from typing import Any, Dict, Sequence import numpy -from openfisca_core.types import ArrayLike, Array - -T = TypeVar("T") - def apply_thresholds( - input: Array[float], - thresholds: ArrayLike[float], - choices: ArrayLike[float], - ) -> Array[float]: + input: numpy.float_, + thresholds: Sequence[float], + choices: Sequence[float], + ) -> numpy.ndarray: """Makes a choice based on an input and thresholds. From a list of ``choices``, this function selects one of these values @@ -33,20 +29,21 @@ def apply_thresholds( Examples: >>> input = numpy.array([4, 5, 6, 7, 8]) - >>> thresholds = [5, 7] + >>> thresholds = + [5, 7] >>> choices = [10, 15, 20] >>> apply_thresholds(input, thresholds, choices) array([10, 10, 15, 15, 20]) """ - condlist: Sequence[Array[bool]] + condlist: Sequence[numpy.bool_] condlist = [input <= threshold for threshold in thresholds] if len(condlist) == len(choices) - 1: # If a choice is provided for input > highest threshold, last condition # must be true to return it. - condlist += [True] + condlist += numpy.array([True]) assert len(condlist) == len(choices), \ " ".join([ @@ -57,7 +54,7 @@ def apply_thresholds( return numpy.select(condlist, choices) -def concat(this: ArrayLike[str], that: ArrayLike[str]) -> Array[str]: +def concat(this: Sequence[str], that: Sequence[str]) -> numpy.ndarray: """Concatenates the values of two arrays. Args: @@ -90,9 +87,9 @@ def concat(this: ArrayLike[str], that: ArrayLike[str]) -> Array[str]: def switch( - conditions: Array[Any], - value_by_condition: Dict[float, T], - ) -> Array[T]: + conditions: numpy.ndarray, + value_by_condition: Dict[float, Any], + ) -> numpy.ndarray: """Mimicks a switch statement. Given an array of conditions, returns an array of the same size, @@ -122,7 +119,7 @@ def switch( condlist = [ conditions == condition - for condition in value_by_condition.keys() + for condition in tuple(value_by_condition.keys()) ] - return numpy.select(condlist, value_by_condition.values()) + return numpy.select(condlist, tuple(value_by_condition.values())) diff --git a/openfisca_core/tracers/computation_log.py b/openfisca_core/tracers/computation_log.py index 8f1b297dfe..4e3d20e074 100644 --- a/openfisca_core/tracers/computation_log.py +++ b/openfisca_core/tracers/computation_log.py @@ -27,6 +27,7 @@ def display( ) -> str: if isinstance(value, EnumArray): value = value.decode_to_str() + raise ValueError(type(value)) return numpy.array2string(value, max_line_width = None) diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py index 699133aecb..4e5833d980 100644 --- a/openfisca_core/types/__init__.py +++ b/openfisca_core/types/__init__.py @@ -5,9 +5,6 @@ and expected behaviours. Official Public API: - * :attr:`.Array` - * ``ArrayLike`` - * :attr:`.Cache` * :attr:`.Entity` * :attr:`.Formula` * :attr:`.Holder` @@ -49,12 +46,7 @@ # Official Public API -from ._data import ( # noqa: F401 - Array, - ArrayLike, - ) - -from ._domain import ( # noqa: F401 +from ._types import ( # noqa: F401 Entity, Formula, Holder, @@ -70,8 +62,6 @@ ) __all__ = [ - "Array", - "ArrayLike", "Entity", "Formula", "Holder", diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py deleted file mode 100644 index ff7066d43a..0000000000 --- a/openfisca_core/types/_data.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Sequence, TypeVar, Union - -from nptyping import types, NDArray as Array - -import numpy - -T = TypeVar("T", bool, bytes, float, int, object, str) - -types._ndarray_meta._Type = Union[type, numpy.dtype, TypeVar] - -ArrayLike = Union[Array[T], Sequence[T]] -""":obj:`typing.Generic`: Type of any castable to :class:`numpy.ndarray`. - -These include any :obj:`numpy.ndarray` and sequences (like -:obj:`list`, :obj:`tuple`, and so on). - -Examples: - >>> ArrayLike[float] - typing.Union[numpy.ndarray, typing.Sequence[float]] - - >>> ArrayLike[str] - typing.Union[numpy.ndarray, typing.Sequence[str]] - -Note: - It is possible since numpy version 1.21 to specify the type of an - array, thanks to `numpy.typing.NDArray`_:: - - from numpy.typing import NDArray - NDArray[numpy.float64] - - `mypy`_ provides `duck type compatibility`_, so an :obj:`int` is - considered to be valid whenever a :obj:`float` is expected. - -Todo: - * Refactor once numpy version >= 1.21 is used. - -.. versionadded:: 35.5.0 - -.. versionchanged:: 35.6.0 - Moved to :mod:`.types` - -.. _mypy: - https://mypy.readthedocs.io/en/stable/ - -.. _duck type compatibility: - https://mypy.readthedocs.io/en/stable/duck_type_compatibility.html - -.. _numpy.typing.NDArray: - https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray - -""" diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_types.py similarity index 100% rename from openfisca_core/types/_domain.py rename to openfisca_core/types/_types.py diff --git a/setup.py b/setup.py index 1ff4a54c36..f2612d44b8 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,6 @@ general_requirements = [ 'dpath >= 1.5.0, < 3.0.0', - 'nptyping == 1.4.4', 'numexpr >= 2.7.0, <= 3.0', 'numpy >= 1.20.3, < 1.23.0', 'psutil >= 5.4.7, < 6.0.0', From 857d86a0f7dfa466ae7dc3307098e01ef9e9c6c9 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 22 Nov 2022 13:17:26 +0100 Subject: [PATCH 6/7] Fix types in commons --- openfisca_core/commons/misc.py | 4 +- openfisca_core/commons/rates.py | 57 +++++++++++------------ openfisca_core/tracers/computation_log.py | 1 - 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index c2edcb25dc..c0dd4ef342 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,6 +1,6 @@ from typing import TypeVar -from openfisca_core.types import Array +import numpy T = TypeVar("T") @@ -43,7 +43,7 @@ def empty_clone(original: T) -> T: return new -def stringify_array(array: Array) -> str: +def stringify_array(array: numpy.ndarray) -> str: """Generates a clean string representation of a numpy array. Args: diff --git a/openfisca_core/commons/rates.py b/openfisca_core/commons/rates.py index e9d67c322a..05a94b0396 100644 --- a/openfisca_core/commons/rates.py +++ b/openfisca_core/commons/rates.py @@ -1,15 +1,13 @@ -from typing import Optional +from typing import Optional, Sequence import numpy -from openfisca_core.types import ArrayLike, Array - def average_rate( - target: Array[float], - varying: ArrayLike[float], - trim: Optional[ArrayLike[float]] = None, - ) -> Array[float]: + target: numpy.ndarray, + varying: Sequence[float], + trim: Optional[Sequence[float]] = None, + ) -> numpy.ndarray: """Computes the average rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -41,32 +39,31 @@ def average_rate( """ - average_rate: Array[float] - - average_rate = 1 - target / varying + rate: numpy.ndarray + rate = 1 - target / varying if trim is not None: - average_rate = numpy.where( - average_rate <= max(trim), - average_rate, + rate = numpy.where( + rate <= max(trim), + rate, numpy.nan, ) - average_rate = numpy.where( - average_rate >= min(trim), - average_rate, + rate = numpy.where( + rate >= min(trim), + rate, numpy.nan, ) - return average_rate + return rate def marginal_rate( - target: Array[float], - varying: Array[float], - trim: Optional[ArrayLike[float]] = None, - ) -> Array[float]: + target: numpy.ndarray, + varying: numpy.ndarray, + trim: Optional[numpy.ndarray] = None, + ) -> numpy.ndarray: """Computes the marginal rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -98,9 +95,9 @@ def marginal_rate( """ - marginal_rate: Array[float] + rate: numpy.ndarray - marginal_rate = ( + rate = ( + 1 - (target[:-1] - target[1:]) / (varying[:-1] - varying[1:]) @@ -108,16 +105,16 @@ def marginal_rate( if trim is not None: - marginal_rate = numpy.where( - marginal_rate <= max(trim), - marginal_rate, + rate = numpy.where( + rate <= max(trim), + rate, numpy.nan, ) - marginal_rate = numpy.where( - marginal_rate >= min(trim), - marginal_rate, + rate = numpy.where( + rate >= min(trim), + rate, numpy.nan, ) - return marginal_rate + return rate diff --git a/openfisca_core/tracers/computation_log.py b/openfisca_core/tracers/computation_log.py index 4e3d20e074..8f1b297dfe 100644 --- a/openfisca_core/tracers/computation_log.py +++ b/openfisca_core/tracers/computation_log.py @@ -27,7 +27,6 @@ def display( ) -> str: if isinstance(value, EnumArray): value = value.decode_to_str() - raise ValueError(type(value)) return numpy.array2string(value, max_line_width = None) From 1d6e22a92fd2b9609472f1abfc6bea9f3751ce01 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 22 Nov 2022 14:43:39 +0100 Subject: [PATCH 7/7] Fix computation log types --- noxfile.py | 14 ++--- openfisca_core/commons/__init__.py | 5 +- openfisca_core/commons/formulas.py | 11 ++-- openfisca_core/commons/misc.py | 35 +++++++++-- openfisca_core/indexed_enums/enum_array.py | 4 +- openfisca_core/populations/population.py | 24 ++++---- openfisca_core/tracers/computation_log.py | 48 ++++++--------- openfisca_core/tracers/flat_trace.py | 44 +++++--------- openfisca_core/tracers/full_tracer.py | 59 ++++++++++--------- openfisca_core/tracers/performance_log.py | 31 +++++----- openfisca_core/tracers/trace_node.py | 28 ++++----- openfisca_core/types/__init__.py | 17 +++++- openfisca_core/types/_data.py | 3 + .../types/{_types.py => _domain.py} | 32 ++++++++-- openfisca_tasks/install.mk | 10 ---- openfisca_tasks/lint.mk | 12 ++-- openfisca_tasks/test_code.mk | 1 - setup.cfg | 1 + setup.py | 2 +- 19 files changed, 202 insertions(+), 179 deletions(-) create mode 100644 openfisca_core/types/_data.py rename openfisca_core/types/{_types.py => _domain.py} (86%) diff --git a/noxfile.py b/noxfile.py index 60625ff135..7886e41569 100644 --- a/noxfile.py +++ b/noxfile.py @@ -7,7 +7,7 @@ @nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "style")) -@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) def style(session, numpy): """Run tests.""" @@ -21,7 +21,7 @@ def style(session, numpy): @nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "docs")) -@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) def docs(session, numpy): """Run tests.""" @@ -35,7 +35,7 @@ def docs(session, numpy): @nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "mypy")) -@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) def mypy(session, numpy): """Run tests.""" @@ -49,7 +49,7 @@ def mypy(session, numpy): @nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "mypy-hxc")) -@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) def mypy_hxc(session, numpy): """Run tests.""" @@ -63,7 +63,7 @@ def mypy_hxc(session, numpy): @nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-core")) -@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) def test_core(session, numpy): """Run tests.""" @@ -79,7 +79,7 @@ def test_core(session, numpy): @nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-country")) -@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) def test_country(session, numpy): """Run tests.""" @@ -95,7 +95,7 @@ def test_country(session, numpy): @nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-extension")) -@nox.parametrize("numpy", ("1.22", "1.20", "1.21")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) def test_extension(session, numpy): """Run tests.""" diff --git a/openfisca_core/commons/__init__.py b/openfisca_core/commons/__init__.py index b3b5d8cbb2..7caa6c397a 100644 --- a/openfisca_core/commons/__init__.py +++ b/openfisca_core/commons/__init__.py @@ -8,6 +8,7 @@ * :func:`.average_rate` * :func:`.concat` * :func:`.empty_clone` + * :func:`.flatten` * :func:`.marginal_rate` * :func:`.stringify_array` * :func:`.switch` @@ -53,11 +54,11 @@ # Official Public API from .formulas import apply_thresholds, concat, switch # noqa: F401 -from .misc import empty_clone, stringify_array # noqa: F401 +from .misc import empty_clone, flatten, stringify_array # noqa: F401 from .rates import average_rate, marginal_rate # noqa: F401 __all__ = ["apply_thresholds", "concat", "switch"] -__all__ = ["empty_clone", "stringify_array", *__all__] +__all__ = ["empty_clone", "flatten", "stringify_array", *__all__] __all__ = ["average_rate", "marginal_rate", *__all__] # Deprecated diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index 188f6f2062..d1da0fc880 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Sequence +from typing import Any, Dict, Sequence, Union import numpy @@ -29,21 +29,20 @@ def apply_thresholds( Examples: >>> input = numpy.array([4, 5, 6, 7, 8]) - >>> thresholds = - [5, 7] + >>> thresholds = [5, 7] >>> choices = [10, 15, 20] >>> apply_thresholds(input, thresholds, choices) array([10, 10, 15, 15, 20]) """ - condlist: Sequence[numpy.bool_] + condlist: Sequence[Union[bool, numpy.bool_]] condlist = [input <= threshold for threshold in thresholds] if len(condlist) == len(choices) - 1: # If a choice is provided for input > highest threshold, last condition # must be true to return it. - condlist += numpy.array([True]) + condlist += [True] assert len(condlist) == len(choices), \ " ".join([ @@ -119,7 +118,7 @@ def switch( condlist = [ conditions == condition - for condition in tuple(value_by_condition.keys()) + for condition in value_by_condition.keys() ] return numpy.select(condlist, tuple(value_by_condition.values())) diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index c0dd4ef342..36619aa7e7 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,12 +1,14 @@ -from typing import TypeVar +from typing import Any, Iterator, Optional, Sequence, TypeVar -import numpy +import itertools + +from openfisca_core import types T = TypeVar("T") def empty_clone(original: T) -> T: - """Creates an empty instance of the same class of the original object. + """Create an empty instance of the same class of the original object. Args: original: An object to clone. @@ -43,8 +45,31 @@ def empty_clone(original: T) -> T: return new -def stringify_array(array: numpy.ndarray) -> str: - """Generates a clean string representation of a numpy array. +def flatten(seqs: Sequence[Sequence[T]]) -> Iterator[T]: + """Flatten a sequence of sequences. + + Args: + seqs: Any sequence of sequences. + + Returns: + An iterator with the values. + + Examples: + >>> list(flatten([(1, 2), (3, 4)])) + [1, 2, 3, 4] + + >>> list(flatten(["ab", "cd"])) + ['a', 'b', 'c', 'd'] + + .. versionadded:: 36.0.0 + + """ + + return itertools.chain.from_iterable(seqs) + + +def stringify_array(array: Optional[types.Array[Any]]) -> str: + """Generate a clean string representation of a numpy array. Args: array: An array. diff --git a/openfisca_core/indexed_enums/enum_array.py b/openfisca_core/indexed_enums/enum_array.py index 6a77be57a7..3ffa79b87c 100644 --- a/openfisca_core/indexed_enums/enum_array.py +++ b/openfisca_core/indexed_enums/enum_array.py @@ -63,7 +63,7 @@ def _forbidden_operation(self, other: Any) -> NoReturn: __and__ = _forbidden_operation __or__ = _forbidden_operation - def decode(self) -> numpy.object_: + def decode(self) -> numpy.ndarray: """ Return the array of enum items corresponding to self. @@ -82,7 +82,7 @@ def decode(self) -> numpy.object_: list(self.possible_values), ) - def decode_to_str(self) -> numpy.str_: + def decode_to_str(self) -> numpy.ndarray: """ Return the array of string identifiers corresponding to self. diff --git a/openfisca_core/populations/population.py b/openfisca_core/populations/population.py index cb243aff70..3ec95a4a2f 100644 --- a/openfisca_core/populations/population.py +++ b/openfisca_core/populations/population.py @@ -10,7 +10,7 @@ from openfisca_core import periods, projectors from openfisca_core.holders import Holder, MemoryUsage from openfisca_core.projectors import Projector -from openfisca_core.types import Array, Entity, Period, Role, Simulation +from openfisca_core.types import Entity, Period, Role, Simulation from . import config @@ -21,14 +21,14 @@ class Population: entity: Entity _holders: Dict[str, Holder] count: int - ids: Array[str] + ids: numpy.ndarray def __init__(self, entity: Entity) -> None: self.simulation = None self.entity = entity self._holders = {} self.count = 0 - self.ids = [] + self.ids = numpy.array([]) def clone(self, simulation: Simulation) -> Population: result = Population(self.entity) @@ -38,14 +38,14 @@ def clone(self, simulation: Simulation) -> Population: result.ids = self.ids return result - def empty_array(self) -> Array[float]: + def empty_array(self) -> numpy.ndarray: return numpy.zeros(self.count) def filled_array( self, value: Union[float, bool], dtype: Optional[numpy.dtype] = None, - ) -> Union[Array[float], Array[bool]]: + ) -> numpy.ndarray: return numpy.full(self.count, value, dtype) def __getattr__(self, attribute: str) -> Projector: @@ -64,7 +64,7 @@ def get_index(self, id: str) -> int: def check_array_compatible_with_entity( self, - array: Array[float], + array: numpy.ndarray, ) -> None: if self.count == array.size: return None @@ -95,7 +95,7 @@ def __call__( variable_name: str, period: Optional[Union[int, str, Period]] = None, options: Optional[Sequence[str]] = None, - ) -> Optional[Array[float]]: + ) -> Optional[Sequence[float]]: """ Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. @@ -169,7 +169,7 @@ def get_memory_usage( }) @projectors.projectable - def has_role(self, role: Role) -> Optional[Array[bool]]: + def has_role(self, role: Role) -> Optional[Sequence[bool]]: """ Check if a person has a given role within its `GroupEntity` @@ -195,10 +195,10 @@ def has_role(self, role: Role) -> Optional[Array[bool]]: @projectors.projectable def value_from_partner( self, - array: Array[float], + array: numpy.ndarray, entity: Projector, role: Role, - ) -> Optional[Array[float]]: + ) -> Optional[numpy.ndarray]: self.check_array_compatible_with_entity(array) self.entity.check_role_validity(role) @@ -218,9 +218,9 @@ def value_from_partner( def get_rank( self, entity: Population, - criteria: Array[float], + criteria: Sequence[float], condition: bool = True, - ) -> Array[int]: + ) -> numpy.ndarray: """ Get the rank of a person within an entity according to a criteria. The person with rank 0 has the minimum value of criteria. diff --git a/openfisca_core/tracers/computation_log.py b/openfisca_core/tracers/computation_log.py index 8f1b297dfe..285c094f38 100644 --- a/openfisca_core/tracers/computation_log.py +++ b/openfisca_core/tracers/computation_log.py @@ -1,40 +1,31 @@ from __future__ import annotations -import typing -from typing import List, Optional, Union +from typing import Any, Optional, Sequence -import numpy - -from .. import tracers -from openfisca_core.indexed_enums import EnumArray +import sys -if typing.TYPE_CHECKING: - from numpy.typing import ArrayLike +import numpy - Array = Union[EnumArray, ArrayLike] +from openfisca_core import commons, types class ComputationLog: + _full_tracer: types.FullTracer - _full_tracer: tracers.FullTracer - - def __init__(self, full_tracer: tracers.FullTracer) -> None: + def __init__(self, full_tracer: types.FullTracer) -> None: self._full_tracer = full_tracer - def display( - self, - value: Optional[Array], - ) -> str: - if isinstance(value, EnumArray): + def display(self, value: types.Array[Any]) -> str: + if isinstance(value, types.EnumArray): value = value.decode_to_str() - return numpy.array2string(value, max_line_width = None) + return numpy.array2string(value, max_line_width = sys.maxsize) def lines( self, aggregate: bool = False, max_depth: Optional[int] = None, - ) -> List[str]: + ) -> Sequence[str]: depth = 1 lines_by_tree = [ @@ -43,7 +34,7 @@ def lines( in self._full_tracer.trees ] - return self._flatten(lines_by_tree) + return tuple(commons.flatten(lines_by_tree)) def print_log(self, aggregate = False, max_depth = None) -> None: """ @@ -67,11 +58,14 @@ def print_log(self, aggregate = False, max_depth = None) -> None: def _get_node_log( self, - node: tracers.TraceNode, + node: types.TraceNode, depth: int, aggregate: bool, max_depth: Optional[int], - ) -> List[str]: + ) -> Sequence[str]: + + node_log: Sequence[str] + children_log: Sequence[Sequence[str]] if max_depth is not None and depth > max_depth: return [] @@ -84,12 +78,12 @@ def _get_node_log( in node.children ] - return node_log + self._flatten(children_logs) + return [*node_log, *commons.flatten(children_logs)] def _print_line( self, depth: int, - node: tracers.TraceNode, + node: types.TraceNode, aggregate: bool, max_depth: Optional[int], ) -> str: @@ -114,9 +108,3 @@ def _print_line( formatted_value = self.display(value) return f"{indent}{node.name}<{node.period}> >> {formatted_value}" - - def _flatten( - self, - lists: List[List[str]], - ) -> List[str]: - return [item for list_ in lists for item in list_] diff --git a/openfisca_core/tracers/flat_trace.py b/openfisca_core/tracers/flat_trace.py index d51dd2576b..4d03f4eab1 100644 --- a/openfisca_core/tracers/flat_trace.py +++ b/openfisca_core/tracers/flat_trace.py @@ -1,28 +1,19 @@ from __future__ import annotations -import typing -from typing import Dict, Optional, Union +from typing import Any, Dict import numpy -from openfisca_core import tracers -from openfisca_core.indexed_enums import EnumArray - -if typing.TYPE_CHECKING: - from numpy.typing import ArrayLike - - Array = Union[EnumArray, ArrayLike] - Trace = Dict[str, dict] +from openfisca_core import indexed_enums as enums, types class FlatTrace: + _full_tracer: types.FullTracer - _full_tracer: tracers.FullTracer - - def __init__(self, full_tracer: tracers.FullTracer) -> None: + def __init__(self, full_tracer: types.FullTracer) -> None: self._full_tracer = full_tracer - def key(self, node: tracers.TraceNode) -> str: + def key(self, node: types.TraceNode) -> str: name = node.name period = node.period return f"{name}<{period}>" @@ -52,26 +43,19 @@ def get_serialized_trace(self) -> dict: for key, flat_trace in self.get_trace().items() } - def serialize( - self, - value: Optional[Array], - ) -> Union[Optional[Array], list]: - if isinstance(value, EnumArray): - value = value.decode_to_str() + def serialize(self, value: Any) -> Any: + if not isinstance(value, numpy.ndarray): + return value - if isinstance(value, numpy.ndarray) and \ - numpy.issubdtype(value.dtype, numpy.dtype(bytes)): - value = value.astype(numpy.dtype(str)) + if isinstance(value, enums.EnumArray): + return value.decode_to_str().tolist() - if isinstance(value, numpy.ndarray): - value = value.tolist() + if numpy.issubdtype(value.dtype, numpy.dtype(bytes)): + return value.astype(numpy.dtype(str)).tolist() - return value + return value.tolist() - def _get_flat_trace( - self, - node: tracers.TraceNode, - ) -> Trace: + def _get_flat_trace(self, node: types.TraceNode) -> Dict[str, dict]: key = self.key(node) node_trace = { diff --git a/openfisca_core/tracers/full_tracer.py b/openfisca_core/tracers/full_tracer.py index 6638a789d4..29c90547c6 100644 --- a/openfisca_core/tracers/full_tracer.py +++ b/openfisca_core/tracers/full_tracer.py @@ -1,34 +1,34 @@ from __future__ import annotations import time -import typing -from typing import Dict, Iterator, List, Optional, Union +from typing import Any, Dict, Iterator, List, Optional, Sequence, Union -from .. import tracers +from openfisca_core import types -if typing.TYPE_CHECKING: - from numpy.typing import ArrayLike +from .computation_log import ComputationLog +from .flat_trace import FlatTrace +from .performance_log import PerformanceLog +from .simple_tracer import SimpleTracer +from .trace_node import TraceNode - from openfisca_core.periods import Period - - Stack = List[Dict[str, Union[str, Period]]] +Stack = List[Dict[str, Union[str, types.Period]]] +Value = Union[types.Array[Any], Sequence[Any]] class FullTracer: - - _simple_tracer: tracers.SimpleTracer + _simple_tracer: SimpleTracer _trees: list - _current_node: Optional[tracers.TraceNode] + _current_node: Optional[types.TraceNode] def __init__(self) -> None: - self._simple_tracer = tracers.SimpleTracer() + self._simple_tracer = SimpleTracer() self._trees = [] self._current_node = None def record_calculation_start( self, variable: str, - period: Period, + period: types.Period, ) -> None: self._simple_tracer.record_calculation_start(variable, period) self._enter_calculation(variable, period) @@ -37,9 +37,9 @@ def record_calculation_start( def _enter_calculation( self, variable: str, - period: Period, + period: types.Period, ) -> None: - new_node = tracers.TraceNode( + new_node = TraceNode( name = variable, period = period, parent = self._current_node, @@ -56,13 +56,14 @@ def _enter_calculation( def record_parameter_access( self, parameter: str, - period: Period, - value: ArrayLike, + period: types.Period, + value: Value, ) -> None: if self._current_node is not None: - self._current_node.parameters.append( - tracers.TraceNode(name = parameter, period = period, value = value), + self._current_node.parameters = ( + *self._current_node.parameters, + TraceNode(parameter, period, value = value), ) def _record_start_time( @@ -75,7 +76,7 @@ def _record_start_time( if self._current_node is not None: self._current_node.start = time_in_s - def record_calculation_result(self, value: ArrayLike) -> None: + def record_calculation_result(self, value: Value) -> None: if self._current_node is not None: self._current_node.value = value @@ -103,20 +104,20 @@ def stack(self) -> Stack: return self._simple_tracer.stack @property - def trees(self) -> List[tracers.TraceNode]: + def trees(self) -> List[TraceNode]: return self._trees @property - def computation_log(self) -> tracers.ComputationLog: - return tracers.ComputationLog(self) + def computation_log(self) -> ComputationLog: + return ComputationLog(self) @property - def performance_log(self) -> tracers.PerformanceLog: - return tracers.PerformanceLog(self) + def performance_log(self) -> PerformanceLog: + return PerformanceLog(self) @property - def flat_trace(self) -> tracers.FlatTrace: - return tracers.FlatTrace(self) + def flat_trace(self) -> FlatTrace: + return FlatTrace(self) def _get_time_in_sec(self) -> float: return time.time_ns() / (10**9) @@ -130,7 +131,7 @@ def generate_performance_graph(self, dir_path: str) -> None: def generate_performance_tables(self, dir_path: str) -> None: self.performance_log.generate_performance_tables(dir_path) - def _get_nb_requests(self, tree: tracers.TraceNode, variable: str) -> int: + def _get_nb_requests(self, tree: TraceNode, variable: str) -> int: tree_call = tree.name == variable children_calls = sum( self._get_nb_requests(child, variable) @@ -153,7 +154,7 @@ def get_flat_trace(self) -> dict: def get_serialized_flat_trace(self) -> dict: return self.flat_trace.get_serialized_trace() - def browse_trace(self) -> Iterator[tracers.TraceNode]: + def browse_trace(self) -> Iterator[TraceNode]: def _browse_node(node): yield node diff --git a/openfisca_core/tracers/performance_log.py b/openfisca_core/tracers/performance_log.py index 754d7f8056..f9a9066c47 100644 --- a/openfisca_core/tracers/performance_log.py +++ b/openfisca_core/tracers/performance_log.py @@ -1,23 +1,24 @@ from __future__ import annotations +from typing import Dict, Tuple, Sequence + import csv import importlib.resources import itertools import json import os -import typing -from .. import tracers +from openfisca_core import types -if typing.TYPE_CHECKING: - Trace = typing.Dict[str, dict] - Calculation = typing.Tuple[str, dict] - SortedTrace = typing.List[Calculation] +from .trace_node import TraceNode +Trace = Dict[str, dict] +Calculation = Tuple[str, dict] +SortedTrace = Sequence[Calculation] -class PerformanceLog: - def __init__(self, full_tracer: tracers.FullTracer) -> None: +class PerformanceLog: + def __init__(self, full_tracer: types.FullTracer) -> None: self._full_tracer = full_tracer def generate_graph(self, dir_path: str) -> None: @@ -66,7 +67,7 @@ def generate_performance_tables(self, dir_path: str) -> None: def aggregate_calculation_times( self, flat_trace: Trace, - ) -> typing.Dict[str, dict]: + ) -> Dict[str, dict]: def _aggregate_calculations(calculations: list) -> dict: calculation_count = len(calculations) @@ -85,10 +86,10 @@ def _aggregate_calculations(calculations: list) -> dict: return { 'calculation_count': calculation_count, - 'calculation_time': tracers.TraceNode.round(calculation_time), - 'formula_time': tracers.TraceNode.round(formula_time), - 'avg_calculation_time': tracers.TraceNode.round(calculation_time / calculation_count), - 'avg_formula_time': tracers.TraceNode.round(formula_time / calculation_count), + 'calculation_time': TraceNode.round(calculation_time), + 'formula_time': TraceNode.round(formula_time), + 'avg_calculation_time': TraceNode.round(calculation_time / calculation_count), + 'avg_formula_time': TraceNode.round(formula_time / calculation_count), } def _groupby(calculation: Calculation) -> str: @@ -112,7 +113,7 @@ def _json(self) -> dict: 'children': children, } - def _json_tree(self, tree: tracers.TraceNode) -> dict: + def _json_tree(self, tree: types.TraceNode) -> dict: calculation_total_time = tree.calculation_time() children = [self._json_tree(child) for child in tree.children] @@ -122,7 +123,7 @@ def _json_tree(self, tree: tracers.TraceNode) -> dict: 'children': children, } - def _write_csv(self, path: str, rows: typing.List[dict]) -> None: + def _write_csv(self, path: str, rows: Sequence[dict]) -> None: fieldnames = list(rows[0].keys()) with open(path, 'w') as csv_file: diff --git a/openfisca_core/tracers/trace_node.py b/openfisca_core/tracers/trace_node.py index 93b630886c..9190198645 100644 --- a/openfisca_core/tracers/trace_node.py +++ b/openfisca_core/tracers/trace_node.py @@ -1,30 +1,24 @@ from __future__ import annotations -import dataclasses -import typing - -if typing.TYPE_CHECKING: - import numpy +from typing import Any, Optional, Sequence, Union - from openfisca_core.indexed_enums import EnumArray - from openfisca_core.periods import Period +import dataclasses - Array = typing.Union[EnumArray, numpy.typing.ArrayLike] - Time = typing.Union[float, int] +from openfisca_core import types @dataclasses.dataclass class TraceNode: name: str - period: Period - parent: typing.Optional[TraceNode] = None - children: typing.List[TraceNode] = dataclasses.field(default_factory = list) - parameters: typing.List[TraceNode] = dataclasses.field(default_factory = list) - value: typing.Optional[Array] = None + period: types.Period + parent: Optional[TraceNode] = None + children: Sequence[TraceNode] = dataclasses.field(default_factory = list) + parameters: Sequence[TraceNode] = dataclasses.field(default_factory = list) + value: Optional[Union[types.Array[Any], Sequence[Any]]] = None start: float = 0 end: float = 0 - def calculation_time(self, round_: bool = True) -> Time: + def calculation_time(self, round_: bool = True) -> float: result = self.end - self.start if round_: @@ -47,8 +41,8 @@ def formula_time(self) -> float: return self.round(result) def append_child(self, node: TraceNode) -> None: - self.children.append(node) + self.children = (*self.children, node) @staticmethod - def round(time: Time) -> float: + def round(time: float) -> float: return float(f'{time:.4g}') # Keep only 4 significant figures diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py index 4e5833d980..31c7e340b1 100644 --- a/openfisca_core/types/__init__.py +++ b/openfisca_core/types/__init__.py @@ -5,8 +5,11 @@ and expected behaviours. Official Public API: + * :attr:`.Array` * :attr:`.Entity` + * :attr:`.EnumArray` * :attr:`.Formula` + * :attr:`.FullTracer` * :attr:`.Holder` * :attr:`.Instant` * :attr:`.ParameterNodeAtInstant` @@ -16,6 +19,7 @@ * :attr:`.Role`, * :attr:`.Simulation`, * :attr:`.TaxBenefitSystem` + * :attr:`.TraceNode` * :attr:`.Variable` Note: @@ -46,9 +50,15 @@ # Official Public API -from ._types import ( # noqa: F401 +from ._data import ( # noqa: F401 + Array, + ) + +from ._domain import ( # noqa: F401 Entity, + EnumArray, Formula, + FullTracer, Holder, Instant, ParameterNodeAtInstant, @@ -58,12 +68,16 @@ Role, Simulation, TaxBenefitSystem, + TraceNode, Variable, ) __all__ = [ + "Array", "Entity", + "EnumArray", "Formula", + "FullTracer", "Holder", "Instant", "ParameterNodeAtInstant", @@ -73,5 +87,6 @@ "Role", "Simulation", "TaxBenefitSystem", + "TraceNode", "Variable", ] diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py new file mode 100644 index 0000000000..572809207f --- /dev/null +++ b/openfisca_core/types/_data.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from numpy.typing import NDArray as Array # noqa: F401 diff --git a/openfisca_core/types/_types.py b/openfisca_core/types/_domain.py similarity index 86% rename from openfisca_core/types/_types.py rename to openfisca_core/types/_domain.py index 643f27964f..0b61676635 100644 --- a/openfisca_core/types/_types.py +++ b/openfisca_core/types/_domain.py @@ -10,7 +10,6 @@ class Entity(Protocol): """Entity protocol.""" - key: Any plural: Any @@ -30,6 +29,15 @@ def get_variable( """Abstract method.""" +@typing_extensions.runtime_checkable +class EnumArray(Protocol): + """EnumArray protocol.""" + + @abc.abstractmethod + def decode_to_str(self) -> numpy.ndarray: + """Abstract method.""" + + class Formula(Protocol): """Formula protocol.""" @@ -43,6 +51,12 @@ def __call__( """Abstract method.""" +class FullTracer(Protocol): + """FullTracer protocol.""" + trees: Any + browse_trace: Any + + class Holder(Protocol): """Holder protocol.""" @@ -80,6 +94,7 @@ class Period(Protocol): @abc.abstractmethod def start(self) -> Any: """Abstract method.""" + @property @abc.abstractmethod def unit(self) -> Any: @@ -88,7 +103,6 @@ def unit(self) -> Any: class Population(Protocol): """Population protocol.""" - entity: Any @abc.abstractmethod @@ -98,7 +112,6 @@ def get_holder(self, variable_name: Any) -> Any: class Role(Protocol): """Role protocol.""" - entity: Any subroles: Any @@ -125,7 +138,6 @@ def get_population(self, plural: Optional[Any]) -> Any: class TaxBenefitSystem(Protocol): """TaxBenefitSystem protocol.""" - person_entity: Any @abc.abstractmethod @@ -136,7 +148,17 @@ def get_variable( """Abstract method.""" +class TraceNode(Protocol): + """TraceNode protocol.""" + name: Any + value: Any + period: Any + children: Any + parameters: Any + formula_time: Any + calculation_time: Any + + class Variable(Protocol): """Variable protocol.""" - entity: Any diff --git a/openfisca_tasks/install.mk b/openfisca_tasks/install.mk index d047a6eb15..50f0a20b55 100644 --- a/openfisca_tasks/install.mk +++ b/openfisca_tasks/install.mk @@ -3,16 +3,6 @@ uninstall: @$(call print_help,$@:) @pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs pip uninstall -y -## Install project's overall dependencies -install-deps: - @$(call print_help,$@:) - @pip install --upgrade pip twine wheel - -## Install project's development dependencies. -install-edit: - @$(call print_help,$@:) - @pip install --upgrade --editable ".[dev]" - ## Install project's overall dependencies install-deps: @$(call print_help,$@:) diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 115c6267bb..5e1a6f3d73 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -11,7 +11,7 @@ check-syntax-errors: . ## Run linters to check for syntax and style errors. check-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @flake8 $? + @python -m flake8 $? @$(call print_pass,$@:) ## Run linters to check for syntax and style errors in the doc. @@ -29,14 +29,14 @@ lint-doc-%: @## able to integrate documentation improvements progresively. @## @$(call print_help,$(subst $*,%,$@:)) - @flake8 --select=D101,D102,D103,DAR openfisca_core/$* - @pylint openfisca_core/$* + @python -m flake8 --select=D101,D102,D103,DAR openfisca_core/$* + @python -m pylint openfisca_core/$* @$(call print_pass,$@:) ## Run static type checkers for type errors. check-types: @$(call print_help,$@:) - @mypy --package openfisca_core --package openfisca_web_api + @python -m mypy --package openfisca_core --package openfisca_web_api @$(call print_pass,$@:) ## Run static type checkers for type errors (strict). @@ -48,7 +48,7 @@ lint-typing-strict: \ ## Run static type checkers for type errors (strict). lint-typing-strict-%: @$(call print_help,$(subst $*,%,$@:)) - @mypy \ + @python -m mypy \ --cache-dir .mypy_cache-openfisca_core.$* \ --implicit-reexport \ --strict \ @@ -58,5 +58,5 @@ lint-typing-strict-%: ## Run code formatters to correct style errors. format-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @autopep8 $? + @python -m autopep8 $? @$(call print_pass,$@:) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 703cf4ec20..fe83c187f6 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -4,7 +4,6 @@ openfisca = openfisca_core.scripts.openfisca_command ## The path to the installed packages. python_packages = $(shell python -c "import sysconfig; print(sysconfig.get_paths()[\"purelib\"])") -<<<<<<< HEAD ## Run all tasks required for testing. install: install-deps install-edit install-test diff --git a/setup.cfg b/setup.cfg index 467e3ede59..651df9dda4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,6 +47,7 @@ testpaths = tests ignore_missing_imports = True install_types = True non_interactive = True +plugins = numpy.typing.mypy_plugin [mypy-openfisca_core.commons.tests.*] ignore_errors = True diff --git a/setup.py b/setup.py index f2612d44b8..82b53fa731 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ general_requirements = [ 'dpath >= 1.5.0, < 3.0.0', 'numexpr >= 2.7.0, <= 3.0', - 'numpy >= 1.20.3, < 1.23.0', + 'numpy >= 1.21.6, < 1.23.0', 'psutil >= 5.4.7, < 6.0.0', 'pytest >= 4.4.1, < 6.0.0', # For openfisca test 'PyYAML >= 3.10',