From b8168d289b6cb414c68c11c2235ae3060491c532 Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Sat, 19 Sep 2020 10:13:02 +0200 Subject: [PATCH 01/13] Simplified and corrected sorting of subindicators in the indicator_data_serializer --- .../test_indicator_data_serializer.py | 53 +++++++++++++++++++ .../serializers/indicator_data_serializer.py | 39 ++++++-------- 2 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 tests/profile/serializers/test_indicator_data_serializer.py diff --git a/tests/profile/serializers/test_indicator_data_serializer.py b/tests/profile/serializers/test_indicator_data_serializer.py new file mode 100644 index 00000000..9a87af91 --- /dev/null +++ b/tests/profile/serializers/test_indicator_data_serializer.py @@ -0,0 +1,53 @@ +from collections import OrderedDict + +import pytest + +from wazimap_ng.profile.serializers.indicator_data_serializer import sort_subindicators + +@pytest.fixture +def subindicators(): + return OrderedDict(a="x", b="y", c="z") + +class TestSubindicatorSort: + + def test_correct_sort(self, subindicators): + sort_order = ["b", "c", "a"] + + sorted_subindicators = sort_subindicators(subindicators, sort_order) + assert list(sorted_subindicators.keys()) == sort_order + assert list(sorted_subindicators.values()) == [subindicators[k] for k in sort_order] + + sort_order = ["c", "a", "b"] + + sorted_subindicators = sort_subindicators(subindicators, sort_order) + assert list(sorted_subindicators.keys()) == sort_order + assert list(sorted_subindicators.values()) == [subindicators[k] for k in sort_order] + + def test_sort_with_missing_order(self, subindicators): + sorted_subindicators = sort_subindicators(subindicators, None) + assert list(sorted_subindicators.keys()) == list(subindicators.keys()) + assert list(sorted_subindicators.values()) == list(subindicators.values()) + + sorted_subindicators = sort_subindicators(subindicators, []) + assert list(sorted_subindicators.keys()) == list(subindicators.keys()) + assert list(sorted_subindicators.values()) == list(subindicators.values()) + + def test_sort_with_incomplete_order(self, subindicators): + sort_order = ["b", "c"] + sorted_subindicators = sort_subindicators(subindicators, sort_order) + assert list(sorted_subindicators.keys()) == ["b", "c", "a"] + assert list(sorted_subindicators.values()) == [subindicators[k] for k in ["b", "c", "a"]] + + def test_that_return_dict_is_ordereda(self, subindicators): + sorted_subindicators = sort_subindicators(subindicators, ["a", "c", "b"]) + + + assert type(sorted_subindicators) == OrderedDict + + def test_duplicate_keys_are_ignored(self, subindicators): + sort_order = ["b", "c", "a", "b"] + + sorted_subindicators = sort_subindicators(subindicators, sort_order) + assert list(sorted_subindicators.keys()) == ["b", "c", "a"] + assert list(sorted_subindicators.values()) == [subindicators[k] for k in ["b", "c", "a"]] + diff --git a/wazimap_ng/profile/serializers/indicator_data_serializer.py b/wazimap_ng/profile/serializers/indicator_data_serializer.py index 5e9e5bed..7bdbbc69 100644 --- a/wazimap_ng/profile/serializers/indicator_data_serializer.py +++ b/wazimap_ng/profile/serializers/indicator_data_serializer.py @@ -1,6 +1,7 @@ +import operator import logging -from collections import OrderedDict +from collections import OrderedDict, Counter from django.db.models import F from wazimap_ng.datasets.models import IndicatorData, Group @@ -64,6 +65,14 @@ def get_child_indicator_data(profile, geography): return children_profiles +def sort_subindicators(subindicators, sort_order): + sorted_dict = subindicators + unique_sort_order = list(Counter(sort_order).keys()) + if unique_sort_order is not None: + sorted_tuples = sort_list_using_order(subindicators.items(), unique_sort_order, operator.itemgetter(0)) + sorted_dict = OrderedDict(sorted_tuples) + return sorted_dict + def IndicatorDataSerializer(profile, geography): indicator_data = get_indicator_data(profile, geography) children_indicator_data = get_child_indicator_data(profile, geography) @@ -121,30 +130,16 @@ def sort_group_subindicators(row, group_dict): return new_dict - def sort_indicator_subindicators(row, group_dict): - key = (row["indicator_group"][0], row["dataset"]) - key_func = lambda x: x[0] - - new_group_dict = {} - for group, group_subindicators_dict in group_dict.items(): - new_group_subindicators_dict = {} - for group_subindicator, indicator_subindicators_dict in group_subindicators_dict.items(): - if key in groups_lookup: - subindicator_order = groups_lookup[key] - items = indicator_subindicators_dict.items() - sorted_tuples = sort_list_using_order(items, subindicator_order, key_func=key_func) - sorted_indicator_subindicators_dict = OrderedDict(sorted_tuples) - else: - sorted_indicator_subindicators_dict = indicator_subindicators_dict - new_group_subindicators_dict[group_subindicator] = sorted_indicator_subindicators_dict - new_group_dict[group] = new_group_subindicators_dict - - return new_group_dict - def prepare_json(row): json_data = rearrange_group(row["jsdata"]["groups"]) json_data = sort_group_subindicators(row, json_data) - json_data = sort_indicator_subindicators(row, json_data) + + primary_group = row["indicator_group"][0] + key = (primary_group, row["dataset"]) + sort_order = groups_lookup.get(key, None) + subindicators = row["jsdata"]["subindicators"] + + row["jsdata"]["subindicators"] = sort_subindicators(subindicators, sort_order) return json_data From 9f30c06e4cfcabdf23251efee1bf6c8de13590e3 Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Sat, 19 Sep 2020 11:44:31 +0200 Subject: [PATCH 02/13] Simplified group_subindicator sorting --- .../serializers/indicator_data_serializer.py | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/wazimap_ng/profile/serializers/indicator_data_serializer.py b/wazimap_ng/profile/serializers/indicator_data_serializer.py index 7bdbbc69..1e01f84f 100644 --- a/wazimap_ng/profile/serializers/indicator_data_serializer.py +++ b/wazimap_ng/profile/serializers/indicator_data_serializer.py @@ -73,6 +73,20 @@ def sort_subindicators(subindicators, sort_order): sorted_dict = OrderedDict(sorted_tuples) return sorted_dict +def sort_group_subindicators(group_dict, primary_order, order_lookup): + new_dict = {} + for group, group_subindicators_dict in group_dict.items(): + group_order = order_lookup(group) + sorted_group_subindicators_dict = sort_subindicators(group_subindicators_dict, group_order) + + for group_subindicator, indicator_subindicators in sorted_group_subindicators_dict.items(): + sorted_group_subindicators_dict[group_subindicator] = sort_subindicators(indicator_subindicators, primary_order) + + new_dict[group] = sorted_group_subindicators_dict + + return new_dict + + def IndicatorDataSerializer(profile, geography): indicator_data = get_indicator_data(profile, geography) children_indicator_data = get_child_indicator_data(profile, geography) @@ -113,35 +127,22 @@ def rearrange_group(group_dict): } return group_dict - def sort_group_subindicators(row, group_dict): - new_dict = {} - for group, group_subindicators_dict in group_dict.items(): - key = (group, row["dataset"]) - if key in groups_lookup: - key_func = lambda x: x[0] - subindicator_order = groups_lookup[key] - sorted_group_subindicators_list = sort_list_using_order(group_subindicators_dict.items(), subindicator_order, key_func=key_func) - sorted_group_subindicators_dict = OrderedDict(sorted_group_subindicators_list) - else: - logger.warning(f"Key: {key} not in groups lookup") - sorted_group_subindicators_dict = group_subindicators_dict - - new_dict[group] = sorted_group_subindicators_dict - - return new_dict - def prepare_json(row): - json_data = rearrange_group(row["jsdata"]["groups"]) - json_data = sort_group_subindicators(row, json_data) + dataset = row["dataset"] + groups = row["indicator_group"] + primary_group = groups[0] + + order_lookup = lambda group: groups_lookup.get((group, dataset), None) + primary_order = order_lookup(primary_group) - primary_group = row["indicator_group"][0] - key = (primary_group, row["dataset"]) - sort_order = groups_lookup.get(key, None) + group_data = row["jsdata"]["groups"] + group_data = rearrange_group(group_data) subindicators = row["jsdata"]["subindicators"] - row["jsdata"]["subindicators"] = sort_subindicators(subindicators, sort_order) + group_data = sort_group_subindicators(group_data, primary_order, order_lookup) + row["jsdata"]["subindicators"] = sort_subindicators(subindicators, primary_order) - return json_data + return group_data d_groups = qsdict(indicator_data, From ed524d8090b9256be520b92a4b347e9f9dbc999e Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Sat, 19 Sep 2020 16:23:02 +0200 Subject: [PATCH 03/13] Extracted SubindicatorSorter as a separate class Easier to test and reason about it if it is extracted from the rest of the business logic --- .../test_indicator_data_serializer.py | 126 ++++++++++++++---- .../serializers/subindicator_sorter.py | 32 +++++ 2 files changed, 129 insertions(+), 29 deletions(-) create mode 100644 wazimap_ng/profile/serializers/subindicator_sorter.py diff --git a/tests/profile/serializers/test_indicator_data_serializer.py b/tests/profile/serializers/test_indicator_data_serializer.py index 9a87af91..bb5f488f 100644 --- a/tests/profile/serializers/test_indicator_data_serializer.py +++ b/tests/profile/serializers/test_indicator_data_serializer.py @@ -2,52 +2,120 @@ import pytest -from wazimap_ng.profile.serializers.indicator_data_serializer import sort_subindicators +from wazimap_ng.profile.serializers.subindicator_sorter import SubindicatorSorter @pytest.fixture def subindicators(): return OrderedDict(a="x", b="y", c="z") -class TestSubindicatorSort: +@pytest.fixture +def group_subindicators(): + return { + "group1": { + "a": {"c1": 10, "c2": 20, "c3": 30}, + "b": {"c1": 100, "c2": 200, "c3": 300}, + "c": {"c1": 1000, "c2": 2000, "c3": 3000} + }, + "group2": { + "x": {"c1": 40, "c2": 50, "c3": 60}, + "y": {"c1": 400, "c2": 500, "c3": 600}, + "z": {"c1": 4000, "c2": 5000, "c3": 6000} + } + } - def test_correct_sort(self, subindicators): +@pytest.fixture +def sorter(): + group_orders = { + "group1": ["b", "c", "a"], + "group2": ["z", "y", "x"], + "group3": ["c2", "c3", "c1"], + } + sorter = SubindicatorSorter(group_orders) + + return sorter + +class TestSubindicatorSorter: + def test_subindicator_sort(self, sorter, subindicators): sort_order = ["b", "c", "a"] - sorted_subindicators = sort_subindicators(subindicators, sort_order) - assert list(sorted_subindicators.keys()) == sort_order - assert list(sorted_subindicators.values()) == [subindicators[k] for k in sort_order] + sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") sort_order = ["c", "a", "b"] + sorter._group_orders["group1"] = sort_order + + sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + assert sorted_subindicators == OrderedDict(c="z", a="x", b="y") + + def test_subindicator_sort_with_missing_order(self, sorter, subindicators): + sorted_subindicators = sorter.sort_subindicators("missing group", subindicators) + assert sorted_subindicators == OrderedDict(a="x", b="y", c="z") + + def test_subindicator_sort_with_incomplete_order(self, sorter, subindicators): + sorter._group_orders["group1"] = ["b", "c"] + sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + + assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") + + def test_subindicator_sort_with_extra_order_element(self, sorter, subindicators): + sorter._group_orders["group1"] = ["b", "c", "a", "z"] + sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + + assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") + + def test_subindicator_duplicate_keys_are_ignored(self, sorter, subindicators): + sorter._group_orders["group1"] = ["b", "c", "a", "b"] + + sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") + + def test_subindicator_no_side_effects(self, sorter, subindicators): + sort_order = ["b", "c", "a"] + + sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + sorted_subindicators["dummy"] = "XXX" + assert "dummy" not in subindicators + + def test_group_sort(self, sorter, group_subindicators): + sorted_groups = sorter.sort_groups(group_subindicators, "group3") + sorted_group1 = sorted_groups["group1"] + sorted_group2 = sorted_groups["group2"] + + assert list(sorted_group1.keys()) == ["b", "c", "a"] - sorted_subindicators = sort_subindicators(subindicators, sort_order) - assert list(sorted_subindicators.keys()) == sort_order - assert list(sorted_subindicators.values()) == [subindicators[k] for k in sort_order] + sorted_group1_group3 = list(sorted_group1.values()) + assert sorted_group1_group3[0] == OrderedDict(c2=200, c3=300, c1=100) + assert sorted_group1_group3[1] == OrderedDict(c2=2000, c3=3000, c1=1000) + assert sorted_group1_group3[2] == OrderedDict(c2=20, c3=30, c1=10) - def test_sort_with_missing_order(self, subindicators): - sorted_subindicators = sort_subindicators(subindicators, None) - assert list(sorted_subindicators.keys()) == list(subindicators.keys()) - assert list(sorted_subindicators.values()) == list(subindicators.values()) + assert list(sorted_group2.keys()) == ["z", "y", "x"] + sorted_group2_group3 = list(sorted_group2.values()) + assert sorted_group2_group3[0] == OrderedDict(c2=5000, c3=6000, c1=4000) + assert sorted_group2_group3[1] == OrderedDict(c2=500, c3=600, c1=400) + assert sorted_group2_group3[2] == OrderedDict(c2=50, c3=60, c1=40) - sorted_subindicators = sort_subindicators(subindicators, []) - assert list(sorted_subindicators.keys()) == list(subindicators.keys()) - assert list(sorted_subindicators.values()) == list(subindicators.values()) + def test_group_subset(self, sorter, group_subindicators): + del group_subindicators["group1"] - def test_sort_with_incomplete_order(self, subindicators): - sort_order = ["b", "c"] - sorted_subindicators = sort_subindicators(subindicators, sort_order) - assert list(sorted_subindicators.keys()) == ["b", "c", "a"] - assert list(sorted_subindicators.values()) == [subindicators[k] for k in ["b", "c", "a"]] + sorted_groups = sorter.sort_groups(group_subindicators, "group3") + assert len(sorted_groups) == 1 - def test_that_return_dict_is_ordereda(self, subindicators): - sorted_subindicators = sort_subindicators(subindicators, ["a", "c", "b"]) + sorted_group1 = sorted_groups["group2"] + assert list(sorted_group1.keys()) == ["z", "y", "x"] + def test_group_empty(self, sorter, group_subindicators): + group_subindicators["group4"] = group_subindicators["group1"] + del group_subindicators["group1"] + del group_subindicators["group2"] - assert type(sorted_subindicators) == OrderedDict + sorted_groups = sorter.sort_groups(group_subindicators, "group3") + assert len(sorted_groups) == 1 - def test_duplicate_keys_are_ignored(self, subindicators): - sort_order = ["b", "c", "a", "b"] + sorted_group4 = sorted_groups["group4"] + assert list(sorted_group4.keys()) == ["a", "b", "c"] - sorted_subindicators = sort_subindicators(subindicators, sort_order) - assert list(sorted_subindicators.keys()) == ["b", "c", "a"] - assert list(sorted_subindicators.values()) == [subindicators[k] for k in ["b", "c", "a"]] + def test_group_no_side_effects(self, sorter, group_subindicators): + sorted_groups = sorter.sort_groups(group_subindicators, "group3") + sorted_groups["dummy"] = "XXX" + assert "dummy" not in group_subindicators diff --git a/wazimap_ng/profile/serializers/subindicator_sorter.py b/wazimap_ng/profile/serializers/subindicator_sorter.py new file mode 100644 index 00000000..285c26c8 --- /dev/null +++ b/wazimap_ng/profile/serializers/subindicator_sorter.py @@ -0,0 +1,32 @@ +from collections import Counter, OrderedDict +import operator + +from wazimap_ng.utils import sort_list_using_order + +class SubindicatorSorter: + def __init__(self, group_orders): + self._group_orders = group_orders + + def sort_subindicators(self, group, subindicators): + sorted_dict = subindicators + + sort_order = self._group_orders.get(group, None) + + if sort_order is not None: + unique_sort_order = list(Counter(sort_order).keys()) + sorted_tuples = sort_list_using_order(subindicators.items(), unique_sort_order, operator.itemgetter(0)) + sorted_dict = OrderedDict(sorted_tuples) + + return sorted_dict + + def sort_groups(self, groups, primary_group): + new_dict = {} + for group, group_subindicators in groups.items(): + sorted_group_subindicators = dict(self.sort_subindicators(group, group_subindicators)) + + for group_subindicator, subindicators in sorted_group_subindicators.items(): + sorted_group_subindicators[group_subindicator] = self.sort_subindicators(primary_group, subindicators) + + new_dict[group] = sorted_group_subindicators + + return new_dict From 7ecbe009b9d208ff6155c49357addd014ccff25a Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Sat, 19 Sep 2020 16:28:58 +0200 Subject: [PATCH 04/13] Extracted subindicator sorting logic from IndicatorDataSerializer --- .../serializers/indicator_data_serializer.py | 48 ++++++------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/wazimap_ng/profile/serializers/indicator_data_serializer.py b/wazimap_ng/profile/serializers/indicator_data_serializer.py index 1e01f84f..d9d3ed31 100644 --- a/wazimap_ng/profile/serializers/indicator_data_serializer.py +++ b/wazimap_ng/profile/serializers/indicator_data_serializer.py @@ -1,13 +1,12 @@ -import operator import logging -from collections import OrderedDict, Counter from django.db.models import F from wazimap_ng.datasets.models import IndicatorData, Group -from wazimap_ng.utils import qsdict, mergedict, expand_nested_list, pivot, sort_list_using_order +from wazimap_ng.utils import qsdict, mergedict, expand_nested_list, pivot from .. import models +from .subindicator_sorter import SubindicatorSorter logger = logging.getLogger(__name__) @@ -64,38 +63,23 @@ def get_child_indicator_data(profile, geography): return children_profiles +def get_sorters(profile): + groups = (Group.objects + .filter(dataset__indicator__profileindicator__profile=profile) + .order_by("dataset") + .values("name", "dataset", "subindicators") + ) -def sort_subindicators(subindicators, sort_order): - sorted_dict = subindicators - unique_sort_order = list(Counter(sort_order).keys()) - if unique_sort_order is not None: - sorted_tuples = sort_list_using_order(subindicators.items(), unique_sort_order, operator.itemgetter(0)) - sorted_dict = OrderedDict(sorted_tuples) - return sorted_dict - -def sort_group_subindicators(group_dict, primary_order, order_lookup): - new_dict = {} - for group, group_subindicators_dict in group_dict.items(): - group_order = order_lookup(group) - sorted_group_subindicators_dict = sort_subindicators(group_subindicators_dict, group_order) - - for group_subindicator, indicator_subindicators in sorted_group_subindicators_dict.items(): - sorted_group_subindicators_dict[group_subindicator] = sort_subindicators(indicator_subindicators, primary_order) - - new_dict[group] = sorted_group_subindicators_dict - - return new_dict + grouped_orders = qsdict(groups, "dataset", "name", "subindicators") + sorters = {ds: SubindicatorSorter(ds_groups) for ds, ds_groups in grouped_orders.items()} + return sorters def IndicatorDataSerializer(profile, geography): indicator_data = get_indicator_data(profile, geography) children_indicator_data = get_child_indicator_data(profile, geography) indicator_data2 = list(expand_nested_list(indicator_data, "jsdata")) - - groups = Group.objects.filter(dataset__indicator__profileindicator__profile=profile).values("name", "dataset", "subindicators") - groups_lookup = { - (x["name"], x["dataset"]): x["subindicators"] for x in groups - } + sorters = get_sorters(profile) subcategories = (models.IndicatorSubcategory.objects.filter(category__profile=profile) .order_by("category__order", "order") @@ -131,16 +115,14 @@ def prepare_json(row): dataset = row["dataset"] groups = row["indicator_group"] primary_group = groups[0] - - order_lookup = lambda group: groups_lookup.get((group, dataset), None) - primary_order = order_lookup(primary_group) + sorter = sorters[dataset] group_data = row["jsdata"]["groups"] group_data = rearrange_group(group_data) subindicators = row["jsdata"]["subindicators"] - group_data = sort_group_subindicators(group_data, primary_order, order_lookup) - row["jsdata"]["subindicators"] = sort_subindicators(subindicators, primary_order) + group_data = sorter.sort_groups(group_data, primary_group) + row["jsdata"]["subindicators"] = sorter.sort_subindicators(primary_group, subindicators) return group_data From 3a6d056d429ddcfe31dd3c6319ffe80e3d76ceab Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 22 Sep 2020 12:43:20 +0200 Subject: [PATCH 05/13] Changed the order of arguments to the sort functions in order for them to be consistent --- .../serializers/test_indicator_data_serializer.py | 14 +++++++------- .../serializers/indicator_data_serializer.py | 2 +- .../profile/serializers/subindicator_sorter.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/profile/serializers/test_indicator_data_serializer.py b/tests/profile/serializers/test_indicator_data_serializer.py index bb5f488f..9befd710 100644 --- a/tests/profile/serializers/test_indicator_data_serializer.py +++ b/tests/profile/serializers/test_indicator_data_serializer.py @@ -38,41 +38,41 @@ class TestSubindicatorSorter: def test_subindicator_sort(self, sorter, subindicators): sort_order = ["b", "c", "a"] - sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + sorted_subindicators = sorter.sort_subindicators(subindicators, "group1") assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") sort_order = ["c", "a", "b"] sorter._group_orders["group1"] = sort_order - sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + sorted_subindicators = sorter.sort_subindicators(subindicators, "group1") assert sorted_subindicators == OrderedDict(c="z", a="x", b="y") def test_subindicator_sort_with_missing_order(self, sorter, subindicators): - sorted_subindicators = sorter.sort_subindicators("missing group", subindicators) + sorted_subindicators = sorter.sort_subindicators(subindicators, "missing group") assert sorted_subindicators == OrderedDict(a="x", b="y", c="z") def test_subindicator_sort_with_incomplete_order(self, sorter, subindicators): sorter._group_orders["group1"] = ["b", "c"] - sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + sorted_subindicators = sorter.sort_subindicators(subindicators, "group1") assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") def test_subindicator_sort_with_extra_order_element(self, sorter, subindicators): sorter._group_orders["group1"] = ["b", "c", "a", "z"] - sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + sorted_subindicators = sorter.sort_subindicators(subindicators, "group1") assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") def test_subindicator_duplicate_keys_are_ignored(self, sorter, subindicators): sorter._group_orders["group1"] = ["b", "c", "a", "b"] - sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + sorted_subindicators = sorter.sort_subindicators(subindicators, "group1") assert sorted_subindicators == OrderedDict(b="y", c="z", a="x") def test_subindicator_no_side_effects(self, sorter, subindicators): sort_order = ["b", "c", "a"] - sorted_subindicators = sorter.sort_subindicators("group1", subindicators) + sorted_subindicators = sorter.sort_subindicators(subindicators, "group1") sorted_subindicators["dummy"] = "XXX" assert "dummy" not in subindicators diff --git a/wazimap_ng/profile/serializers/indicator_data_serializer.py b/wazimap_ng/profile/serializers/indicator_data_serializer.py index d9d3ed31..06df8156 100644 --- a/wazimap_ng/profile/serializers/indicator_data_serializer.py +++ b/wazimap_ng/profile/serializers/indicator_data_serializer.py @@ -122,7 +122,7 @@ def prepare_json(row): subindicators = row["jsdata"]["subindicators"] group_data = sorter.sort_groups(group_data, primary_group) - row["jsdata"]["subindicators"] = sorter.sort_subindicators(primary_group, subindicators) + row["jsdata"]["subindicators"] = sorter.sort_subindicators(subindicators, primary_group) return group_data diff --git a/wazimap_ng/profile/serializers/subindicator_sorter.py b/wazimap_ng/profile/serializers/subindicator_sorter.py index 285c26c8..420cccce 100644 --- a/wazimap_ng/profile/serializers/subindicator_sorter.py +++ b/wazimap_ng/profile/serializers/subindicator_sorter.py @@ -7,7 +7,7 @@ class SubindicatorSorter: def __init__(self, group_orders): self._group_orders = group_orders - def sort_subindicators(self, group, subindicators): + def sort_subindicators(self, subindicators, group): sorted_dict = subindicators sort_order = self._group_orders.get(group, None) @@ -22,10 +22,10 @@ def sort_subindicators(self, group, subindicators): def sort_groups(self, groups, primary_group): new_dict = {} for group, group_subindicators in groups.items(): - sorted_group_subindicators = dict(self.sort_subindicators(group, group_subindicators)) + sorted_group_subindicators = dict(self.sort_subindicators(group_subindicators, group)) for group_subindicator, subindicators in sorted_group_subindicators.items(): - sorted_group_subindicators[group_subindicator] = self.sort_subindicators(primary_group, subindicators) + sorted_group_subindicators[group_subindicator] = self.sort_subindicators(subindicators, primary_group) new_dict[group] = sorted_group_subindicators From 7692804a1ca21e1886205afac92c6d15bc70bb8d Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 22 Sep 2020 13:21:16 +0200 Subject: [PATCH 06/13] Refactored sorting logic into a separate class --- .../serializers/indicator_data_serializer.py | 55 ++++------------ .../serializers/profile_indicator_sorter.py | 65 +++++++++++++++++++ 2 files changed, 76 insertions(+), 44 deletions(-) create mode 100644 wazimap_ng/profile/serializers/profile_indicator_sorter.py diff --git a/wazimap_ng/profile/serializers/indicator_data_serializer.py b/wazimap_ng/profile/serializers/indicator_data_serializer.py index 06df8156..5d37b7ee 100644 --- a/wazimap_ng/profile/serializers/indicator_data_serializer.py +++ b/wazimap_ng/profile/serializers/indicator_data_serializer.py @@ -2,11 +2,11 @@ from django.db.models import F -from wazimap_ng.datasets.models import IndicatorData, Group +from wazimap_ng.datasets.models import IndicatorData from wazimap_ng.utils import qsdict, mergedict, expand_nested_list, pivot from .. import models -from .subindicator_sorter import SubindicatorSorter +from .profile_indicator_sorter import ProfileIndicatorSorter logger = logging.getLogger(__name__) @@ -63,23 +63,19 @@ def get_child_indicator_data(profile, geography): return children_profiles -def get_sorters(profile): - groups = (Group.objects - .filter(dataset__indicator__profileindicator__profile=profile) - .order_by("dataset") - .values("name", "dataset", "subindicators") - ) - - grouped_orders = qsdict(groups, "dataset", "name", "subindicators") - sorters = {ds: SubindicatorSorter(ds_groups) for ds, ds_groups in grouped_orders.items()} - return sorters def IndicatorDataSerializer(profile, geography): + + sorters = ProfileIndicatorSorter(profile) + indicator_data = get_indicator_data(profile, geography) + indicator_data = sorters.sort(indicator_data) + children_indicator_data = get_child_indicator_data(profile, geography) + children_indicator_data = sorters.sort(children_indicator_data) + indicator_data2 = list(expand_nested_list(indicator_data, "jsdata")) - sorters = get_sorters(profile) subcategories = (models.IndicatorSubcategory.objects.filter(category__profile=profile) .order_by("category__order", "order") @@ -98,35 +94,6 @@ def IndicatorDataSerializer(profile, geography): lambda x: {"description": x.description} ) - def rearrange_group(group_dict): - group_dict = dict(group_dict) - for group_subindicators_dict in group_dict.values(): - for subindicator, value_array in group_subindicators_dict.items(): - group_subindicators_dict[subindicator] = {} - for value_dict in value_array: - count = value_dict.pop("count") - value = list(value_dict.values())[0] - group_subindicators_dict[subindicator][value] = { - "count": count - } - return group_dict - - def prepare_json(row): - dataset = row["dataset"] - groups = row["indicator_group"] - primary_group = groups[0] - sorter = sorters[dataset] - - group_data = row["jsdata"]["groups"] - group_data = rearrange_group(group_data) - subindicators = row["jsdata"]["subindicators"] - - group_data = sorter.sort_groups(group_data, primary_group) - row["jsdata"]["subindicators"] = sorter.sort_subindicators(subindicators, primary_group) - - return group_data - - d_groups = qsdict(indicator_data, "category", lambda x: "subcategories", @@ -134,7 +101,7 @@ def prepare_json(row): lambda x: "indicators", "profile_indicator_label", lambda x: "groups", - lambda x: prepare_json(x) + lambda x: x["jsdata"]["groups"] ) d_groups2 = qsdict(children_indicator_data, @@ -146,7 +113,7 @@ def prepare_json(row): lambda x: "groups", lambda x: "children", "geography_code", - lambda x: prepare_json(x) + lambda x: x["jsdata"]["groups"] ) d_groups2 = pivot(d_groups2, [0, 1, 2, 3, 4, 5, 8, 9, 10, 6, 7]) diff --git a/wazimap_ng/profile/serializers/profile_indicator_sorter.py b/wazimap_ng/profile/serializers/profile_indicator_sorter.py new file mode 100644 index 00000000..55bfef38 --- /dev/null +++ b/wazimap_ng/profile/serializers/profile_indicator_sorter.py @@ -0,0 +1,65 @@ +from wazimap_ng.datasets.models import Group +from wazimap_ng.utils import qsdict + +from .subindicator_sorter import SubindicatorSorter + +class ProfileIndicatorSorter: + def __init__(self, profile): + self._sorters = self._get_sorters(profile) + + def _get_sorters(self, profile): + groups = (Group.objects + .filter(dataset__indicator__profileindicator__profile=profile) + .order_by("dataset") + .values("name", "dataset", "subindicators") + ) + + grouped_orders = qsdict(groups, "dataset", "name", "subindicators") + sorters = {ds: SubindicatorSorter(ds_groups) for ds, ds_groups in grouped_orders.items()} + return sorters + + def _rearrange_group(self, group_dict): + group_dict = dict(group_dict) + for group_subindicators_dict in group_dict.values(): + for subindicator, value_array in group_subindicators_dict.items(): + group_subindicators_dict[subindicator] = {} + for value_dict in value_array: + count = value_dict.pop("count") + value = list(value_dict.values())[0] + group_subindicators_dict[subindicator][value] = { + "count": count + } + return group_dict + + def _sort_indicators(self, row, sort_func): + dataset = row["dataset"] + sorter = self._sorters[dataset] + groups = row["indicator_group"] + primary_group = groups[0] + + return sort_func(sorter, primary_group) + + + def sort_groups(self, data): + for row in data: + group_data = row["jsdata"]["groups"] + group_data = self._rearrange_group(group_data) + sort_func = lambda sorter, group: sorter.sort_groups(group_data, group) + + row["jsdata"]["groups"] = self._sort_indicators(row, sort_func) + + yield row + + def sort_subindicators(self, data): + for row in data: + subindicators = row["jsdata"]["subindicators"] + sort_func = lambda sorter, group: sorter.sort_subindicators(subindicators, group) + row["jsdata"]["subindicators"] = self._sort_indicators(row, sort_func) + + yield row + + def sort(self, data): + data = self.sort_groups(data) + data = self.sort_subindicators(data) + + return list(data) From f15a483c5107c538fd975150fe43421c446ebe6e Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 22 Sep 2020 13:24:04 +0200 Subject: [PATCH 07/13] Renamed test to the correct name --- ...t_indicator_data_serializer.py => test_subindicator_sorter.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/profile/serializers/{test_indicator_data_serializer.py => test_subindicator_sorter.py} (100%) diff --git a/tests/profile/serializers/test_indicator_data_serializer.py b/tests/profile/serializers/test_subindicator_sorter.py similarity index 100% rename from tests/profile/serializers/test_indicator_data_serializer.py rename to tests/profile/serializers/test_subindicator_sorter.py From 2d665691d5f80bd40870ca09db681634c5ffc095 Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 22 Sep 2020 16:38:52 +0200 Subject: [PATCH 08/13] Initial tests for profile indicator sorter --- tests/datasets/factoryboy.py | 8 ++ .../test_profile_indicator_sorter.py | 75 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 tests/profile/serializers/test_profile_indicator_sorter.py diff --git a/tests/datasets/factoryboy.py b/tests/datasets/factoryboy.py index 8c9cb3b5..18ddd826 100644 --- a/tests/datasets/factoryboy.py +++ b/tests/datasets/factoryboy.py @@ -40,3 +40,11 @@ class Meta: dataset = factory.SubFactory(DatasetFactory) universe = factory.SubFactory(UniverseFactory) + + +class GroupFactory(factory.django.DjangoModelFactory): + class Meta: + model = models.Group + + dataset = factory.SubFactory(DatasetFactory) + diff --git a/tests/profile/serializers/test_profile_indicator_sorter.py b/tests/profile/serializers/test_profile_indicator_sorter.py new file mode 100644 index 00000000..781f17f2 --- /dev/null +++ b/tests/profile/serializers/test_profile_indicator_sorter.py @@ -0,0 +1,75 @@ +import pytest +import unittest + +from tests.profile import factoryboy as profile_factoryboy +from tests.datasets import factoryboy as datasets_factoryboy + +from wazimap_ng.profile.serializers.profile_indicator_sorter import ProfileIndicatorSorter + +""" +groups = (Group.objects + .filter(dataset__indicator__profileindicator__profile=profile) + .order_by("dataset") + .values("name", "dataset", "subindicators") + ) +return groups + + """ +@pytest.fixture +def profile(): + return profile_factoryboy.ProfileFactory() + +@pytest.fixture +def profile_indicators(profile): + profile_indicator1 = profile_factoryboy.ProfileIndicatorFactory(profile=profile) + profile_indicator2 = profile_factoryboy.ProfileIndicatorFactory(profile=profile) + profile_indicator3 = profile_factoryboy.ProfileIndicatorFactory() + + return [profile_indicator1, profile_indicator2, profile_indicator3] + +@pytest.fixture +def datasets(profile_indicators): + return [ + profile_indicators[0].indicator.dataset, + profile_indicators[1].indicator.dataset, + profile_indicators[2].indicator.dataset, + ] + +@pytest.fixture +def groups(datasets): + subindicators1 = ["g1s1", "g1s2", "g1s3"] + subindicators2 = ["g2s1", "g2s2", "g2s3"] + subindicators3 = ["g3s1", "g3s2", "g3s3"] + subindicators4 = ["a", "b"] + + return [ + datasets_factoryboy.GroupFactory(name="group1", dataset=datasets[0], subindicators=subindicators1), + datasets_factoryboy.GroupFactory(name="group2", dataset=datasets[0], subindicators=subindicators2), + datasets_factoryboy.GroupFactory(name="group3", dataset=datasets[1], subindicators=subindicators3), + datasets_factoryboy.GroupFactory(name="unrelated group", dataset=datasets[2], subindicators=subindicators4), + ] + +@pytest.fixture +def profile_indicator_sorter(profile): + return ProfileIndicatorSorter(profile) + +@pytest.mark.django_db +class TestProfileIndicatorSorter: + def test_create_profile_indicator_sorter(self, datasets, groups, profile_indicator_sorter): + + sorters = profile_indicator_sorter._sorters + + assert len(sorters) == 2 + assert list(sorters.keys()) == [datasets[0].id, datasets[1].id] + + subindicator_sorter1 = sorters[datasets[0].id] + si_groups1 = subindicator_sorter1._group_orders + assert list(si_groups1.keys()) == [groups[0].name, groups[1].name] + assert si_groups1[groups[0].name] == groups[0].subindicators + assert si_groups1[groups[1].name] == groups[1].subindicators + + subindicator_sorter2 = sorters[datasets[1].id] + si_groups2 = subindicator_sorter2._group_orders + assert list(si_groups2.keys()) == [groups[2].name] + assert si_groups2[groups[2].name] == groups[2].subindicators + From 30575d9fa149c73cf9ffaa115af6f3284be1e931 Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 22 Sep 2020 18:56:19 +0200 Subject: [PATCH 09/13] Additional tests for profile indicator sorter --- .../test_profile_indicator_sorter.py | 95 ++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/tests/profile/serializers/test_profile_indicator_sorter.py b/tests/profile/serializers/test_profile_indicator_sorter.py index 781f17f2..61910dc2 100644 --- a/tests/profile/serializers/test_profile_indicator_sorter.py +++ b/tests/profile/serializers/test_profile_indicator_sorter.py @@ -1,5 +1,6 @@ import pytest import unittest +from collections import OrderedDict from tests.profile import factoryboy as profile_factoryboy from tests.datasets import factoryboy as datasets_factoryboy @@ -37,8 +38,8 @@ def datasets(profile_indicators): @pytest.fixture def groups(datasets): - subindicators1 = ["g1s1", "g1s2", "g1s3"] - subindicators2 = ["g2s1", "g2s2", "g2s3"] + subindicators1 = ["g1s3", "g1s2", "g1s1"] + subindicators2 = ["g2s2", "g2s1", "g2s3"] subindicators3 = ["g3s1", "g3s2", "g3s3"] subindicators4 = ["a", "b"] @@ -49,10 +50,74 @@ def groups(datasets): datasets_factoryboy.GroupFactory(name="unrelated group", dataset=datasets[2], subindicators=subindicators4), ] + +@pytest.fixture +def subindicators_data(): + return OrderedDict(a="x", b="y", c="z") + @pytest.fixture -def profile_indicator_sorter(profile): +def profile_indicator_sorter(profile, groups): + """ + groups passed in to make sure that appropriate objects are created + """ return ProfileIndicatorSorter(profile) +@pytest.fixture +def test_data(datasets): + data = [{ + "dataset": datasets[0].id, + "indicator_group": ["group1", "group2"], + "jsdata": { + "subindicators": {"g1s1": "ABC", "g1s2": "DEF", "g1s3": "GHI", }, + "groups": { + "group2": { + "g2s2": [{"group1": "g1s1", "count": 4}, {"group1": "g1s2", "count": 5}, {"group1": "g1s3", "count": 6}], + "g2s1": [{"group1": "g1s1", "count": 1}, {"group1": "g1s2", "count": 2}, {"group1": "g1s3", "count": 3}], + "g2s3": [{"group1": "g1s1", "count": 7}, {"group1": "g1s2", "count": 8}, {"group1": "g1s3", "count": 9}], + } + } + } + }, { + "dataset": datasets[0].id, + "indicator_group": ["group1", "group2"], + "jsdata": { + "subindicators": {"g1s2": "123", "g1s1": "456", "g1s3": "789", }, + "groups": { + "group2": { + "g2s2": [{"group1": "g1s1", "count": 40}, {"group1": "g1s2", "count": 50}, {"group1": "g1s3", "count": 60}], + "g2s1": [{"group1": "g1s1", "count": 10}, {"group1": "g1s2", "count": 20}, {"group1": "g1s3", "count": 30}], + "g2s3": [{"group1": "g1s1", "count": 70}, {"group1": "g1s2", "count": 80}, {"group1": "g1s3", "count": 90}], + } + } + } + }] + + return data + +@pytest.fixture +def expected_subindicators(): + return [ + {"g1s3": "GHI", "g1s2": "DEF", "g1s1": "ABC", }, + {"g1s3": "789", "g1s2": "123", "g1s1": "456", } + ] + +@pytest.fixture +def expected_groups(): + return [{ + "group2": { + "g2s2": OrderedDict(g1s3={"count": 6}, g1s2={"count": 5}, g1s1={"count": 4}), + "g2s1": OrderedDict(g1s3={"count": 3}, g1s2={"count": 2}, g1s1={"count": 1}), + "g2s3": OrderedDict(g1s3={"count": 9}, g1s2={"count": 8}, g1s1={"count": 7}), + } + }, + { + "group2": { + "g2s2": OrderedDict(g1s3={"count": 60}, g1s2={"count": 50}, g1s1={"count": 40}), + "g2s1": OrderedDict(g1s3={"count": 30}, g1s2={"count": 20}, g1s1={"count": 10}), + "g2s3": OrderedDict(g1s3={"count": 90}, g1s2={"count": 80}, g1s1={"count": 70}), + } + }] + @pytest.mark.django_db class TestProfileIndicatorSorter: def test_create_profile_indicator_sorter(self, datasets, groups, profile_indicator_sorter): @@ -73,3 +138,27 @@ def test_create_profile_indicator_sorter(self, datasets, groups, profile_indicat assert list(si_groups2.keys()) == [groups[2].name] assert si_groups2[groups[2].name] == groups[2].subindicators + @pytest.mark.django_db + def test_sort_subindicators(self, profile_indicator_sorter, datasets, test_data, expected_subindicators): + sorted_data = profile_indicator_sorter.sort_subindicators(test_data) + sorted_data = list(sorted_data) + assert sorted_data[0]["jsdata"]["subindicators"] == expected_subindicators[0] + assert sorted_data[1]["jsdata"]["subindicators"] == expected_subindicators[1] + + @pytest.mark.django_db + def test_sort_groups(self, profile_indicator_sorter, test_data, expected_groups): + sorted_data = profile_indicator_sorter.sort_groups(test_data) + sorted_data = list(sorted_data) + assert sorted_data[0]["jsdata"]["groups"] == expected_groups[0] + assert sorted_data[1]["jsdata"]["groups"] == expected_groups[1] + + @pytest.mark.django_db + def test_sort(self, profile_indicator_sorter, test_data, expected_subindicators, expected_groups): + sorted_data = profile_indicator_sorter.sort_groups(test_data) + sorted_data = list(sorted_data) + + assert sorted_data[0]["jsdata"]["subindicators"] == expected_subindicators[0] + assert sorted_data[1]["jsdata"]["subindicators"] == expected_subindicators[1] + + assert sorted_data[0]["jsdata"]["groups"] == expected_groups[0] + assert sorted_data[1]["jsdata"]["groups"] == expected_groups[1] From e810925a1d2579fe561fa6004ea3db4feede23fa Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 22 Sep 2020 19:16:12 +0200 Subject: [PATCH 10/13] Refactored ProfileIndicatorSorter moved the rearrange group code out of the sorter and back into the serializer. The sorter now has a single job. --- .../test_profile_indicator_sorter.py | 26 +++++++++---------- .../serializers/indicator_data_serializer.py | 16 ++++++++++++ .../serializers/profile_indicator_sorter.py | 14 ---------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/tests/profile/serializers/test_profile_indicator_sorter.py b/tests/profile/serializers/test_profile_indicator_sorter.py index 61910dc2..df5a6ac8 100644 --- a/tests/profile/serializers/test_profile_indicator_sorter.py +++ b/tests/profile/serializers/test_profile_indicator_sorter.py @@ -71,9 +71,9 @@ def test_data(datasets): "subindicators": {"g1s1": "ABC", "g1s2": "DEF", "g1s3": "GHI", }, "groups": { "group2": { - "g2s2": [{"group1": "g1s1", "count": 4}, {"group1": "g1s2", "count": 5}, {"group1": "g1s3", "count": 6}], - "g2s1": [{"group1": "g1s1", "count": 1}, {"group1": "g1s2", "count": 2}, {"group1": "g1s3", "count": 3}], - "g2s3": [{"group1": "g1s1", "count": 7}, {"group1": "g1s2", "count": 8}, {"group1": "g1s3", "count": 9}], + "g2s2": {"g1s1": 4, "g1s2": 5, "g1s3": 6}, + "g2s1": {"g1s1": 1, "g1s2": 2, "g1s3": 3}, + "g2s3": {"g1s1": 7, "g1s2": 8, "g1s3": 9}, } } } @@ -84,9 +84,9 @@ def test_data(datasets): "subindicators": {"g1s2": "123", "g1s1": "456", "g1s3": "789", }, "groups": { "group2": { - "g2s2": [{"group1": "g1s1", "count": 40}, {"group1": "g1s2", "count": 50}, {"group1": "g1s3", "count": 60}], - "g2s1": [{"group1": "g1s1", "count": 10}, {"group1": "g1s2", "count": 20}, {"group1": "g1s3", "count": 30}], - "g2s3": [{"group1": "g1s1", "count": 70}, {"group1": "g1s2", "count": 80}, {"group1": "g1s3", "count": 90}], + "g2s2": {"g1s1": 40, "g1s2": 50, "g1s3": 60}, + "g2s1": {"g1s1": 10, "g1s2": 20, "g1s3": 30}, + "g2s3": {"g1s1": 70, "g1s2": 80, "g1s3": 90}, } } } @@ -105,16 +105,16 @@ def expected_subindicators(): def expected_groups(): return [{ "group2": { - "g2s2": OrderedDict(g1s3={"count": 6}, g1s2={"count": 5}, g1s1={"count": 4}), - "g2s1": OrderedDict(g1s3={"count": 3}, g1s2={"count": 2}, g1s1={"count": 1}), - "g2s3": OrderedDict(g1s3={"count": 9}, g1s2={"count": 8}, g1s1={"count": 7}), + "g2s2": OrderedDict(g1s3=6, g1s2=5, g1s1=4), + "g2s1": OrderedDict(g1s3=3, g1s2=2, g1s1=1), + "g2s3": OrderedDict(g1s3=9, g1s2=8, g1s1=7), } }, { "group2": { - "g2s2": OrderedDict(g1s3={"count": 60}, g1s2={"count": 50}, g1s1={"count": 40}), - "g2s1": OrderedDict(g1s3={"count": 30}, g1s2={"count": 20}, g1s1={"count": 10}), - "g2s3": OrderedDict(g1s3={"count": 90}, g1s2={"count": 80}, g1s1={"count": 70}), + "g2s2": OrderedDict(g1s3=60, g1s2=50, g1s1=40), + "g2s1": OrderedDict(g1s3=30, g1s2=20, g1s1=10), + "g2s3": OrderedDict(g1s3=90, g1s2=80, g1s1=70), } }] @@ -159,6 +159,6 @@ def test_sort(self, profile_indicator_sorter, test_data, expected_subindicators, assert sorted_data[0]["jsdata"]["subindicators"] == expected_subindicators[0] assert sorted_data[1]["jsdata"]["subindicators"] == expected_subindicators[1] - + assert sorted_data[0]["jsdata"]["groups"] == expected_groups[0] assert sorted_data[1]["jsdata"]["groups"] == expected_groups[1] diff --git a/wazimap_ng/profile/serializers/indicator_data_serializer.py b/wazimap_ng/profile/serializers/indicator_data_serializer.py index 5d37b7ee..42ad7d4c 100644 --- a/wazimap_ng/profile/serializers/indicator_data_serializer.py +++ b/wazimap_ng/profile/serializers/indicator_data_serializer.py @@ -63,6 +63,20 @@ def get_child_indicator_data(profile, geography): return children_profiles +def rearrange_group(data): + for row in data: + group_dict = row["jsdata"]["groups"] + + for group_subindicators_dict in group_dict.values(): + for subindicator, value_array in group_subindicators_dict.items(): + group_subindicators_dict[subindicator] = {} + for value_dict in value_array: + count = value_dict.pop("count") + value = list(value_dict.values())[0] + group_subindicators_dict[subindicator][value] = { + "count": count + } + yield row def IndicatorDataSerializer(profile, geography): @@ -70,9 +84,11 @@ def IndicatorDataSerializer(profile, geography): sorters = ProfileIndicatorSorter(profile) indicator_data = get_indicator_data(profile, geography) + indicator_data = rearrange_group(indicator_data) indicator_data = sorters.sort(indicator_data) children_indicator_data = get_child_indicator_data(profile, geography) + children_indicator_data = rearrange_group(children_indicator_data) children_indicator_data = sorters.sort(children_indicator_data) indicator_data2 = list(expand_nested_list(indicator_data, "jsdata")) diff --git a/wazimap_ng/profile/serializers/profile_indicator_sorter.py b/wazimap_ng/profile/serializers/profile_indicator_sorter.py index 55bfef38..8c1237ae 100644 --- a/wazimap_ng/profile/serializers/profile_indicator_sorter.py +++ b/wazimap_ng/profile/serializers/profile_indicator_sorter.py @@ -18,19 +18,6 @@ def _get_sorters(self, profile): sorters = {ds: SubindicatorSorter(ds_groups) for ds, ds_groups in grouped_orders.items()} return sorters - def _rearrange_group(self, group_dict): - group_dict = dict(group_dict) - for group_subindicators_dict in group_dict.values(): - for subindicator, value_array in group_subindicators_dict.items(): - group_subindicators_dict[subindicator] = {} - for value_dict in value_array: - count = value_dict.pop("count") - value = list(value_dict.values())[0] - group_subindicators_dict[subindicator][value] = { - "count": count - } - return group_dict - def _sort_indicators(self, row, sort_func): dataset = row["dataset"] sorter = self._sorters[dataset] @@ -43,7 +30,6 @@ def _sort_indicators(self, row, sort_func): def sort_groups(self, data): for row in data: group_data = row["jsdata"]["groups"] - group_data = self._rearrange_group(group_data) sort_func = lambda sorter, group: sorter.sort_groups(group_data, group) row["jsdata"]["groups"] = self._sort_indicators(row, sort_func) From 1d2a826edd80d5c0215d0577cda76515acd34745 Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 22 Sep 2020 19:31:55 +0200 Subject: [PATCH 11/13] removed some junk --- .../profile/serializers/test_profile_indicator_sorter.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/profile/serializers/test_profile_indicator_sorter.py b/tests/profile/serializers/test_profile_indicator_sorter.py index df5a6ac8..81d53e27 100644 --- a/tests/profile/serializers/test_profile_indicator_sorter.py +++ b/tests/profile/serializers/test_profile_indicator_sorter.py @@ -7,15 +7,6 @@ from wazimap_ng.profile.serializers.profile_indicator_sorter import ProfileIndicatorSorter -""" -groups = (Group.objects - .filter(dataset__indicator__profileindicator__profile=profile) - .order_by("dataset") - .values("name", "dataset", "subindicators") - ) -return groups - - """ @pytest.fixture def profile(): return profile_factoryboy.ProfileFactory() From 06f1e94074f172bdf50a2a7335d5d33cef984a50 Mon Sep 17 00:00:00 2001 From: milafrerichs Date: Tue, 29 Sep 2020 12:07:25 +0200 Subject: [PATCH 12/13] feat: add view test for profile geography data to make sure that the indicator data is sorted properly this is a expected failing test to make sure the resulting group is ordered --- tests/datasets/factoryboy.py | 8 +++ tests/profile/test_views.py | 98 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 tests/profile/test_views.py diff --git a/tests/datasets/factoryboy.py b/tests/datasets/factoryboy.py index 18ddd826..330bd6fe 100644 --- a/tests/datasets/factoryboy.py +++ b/tests/datasets/factoryboy.py @@ -42,6 +42,14 @@ class Meta: universe = factory.SubFactory(UniverseFactory) +class IndicatorDataFactory(factory.django.DjangoModelFactory): + class Meta: + model = models.IndicatorData + + indicator = factory.SubFactory(IndicatorFactory) + geography = factory.SubFactory(GeographyFactory) + + class GroupFactory(factory.django.DjangoModelFactory): class Meta: model = models.Group diff --git a/tests/profile/test_views.py b/tests/profile/test_views.py new file mode 100644 index 00000000..b6322b0b --- /dev/null +++ b/tests/profile/test_views.py @@ -0,0 +1,98 @@ +import pytest +from collections import OrderedDict + +from rest_framework.test import APIClient +from django.urls import reverse + +from tests.profile import factoryboy as profile_factoryboy +from tests.datasets import factoryboy as datasets_factoryboy + +@pytest.fixture +def api_client(): + return APIClient() + +@pytest.fixture +def profile(): + return profile_factoryboy.ProfileFactory() + +@pytest.fixture +def indicator_data_items_data(): + return { + "groups": { + "age group": { + "20-24": [ + { + "count": 2.2397, + "gender": "F" + } + ], + "15-19": [ + { + "count": 9.62006, + "gender": "F" + }, + { + "count": 8.79722, + "gender": "M" + } + ], + } + }, + "subindicators": { "M": 52.34565, "F": 56.0179 } + } + +@pytest.fixture +def expected_age_groups(): + return OrderedDict([ + ('15-19',OrderedDict( + M={"count": 8.79722}, + F={"count": 9.62006}, + )), + ('20-24',OrderedDict(F={"count": 2.2397})) + ]) + +@pytest.mark.django_db +class TestProfileGeographyData: + + def test_profile_geography_data_ordering_is_correct(self, api_client, profile, indicator_data_items_data, expected_age_groups): + dataset = datasets_factoryboy.DatasetFactory(geography_hierarchy=profile.geography_hierarchy, groups=["age group", "gender"]) + indicator = datasets_factoryboy.IndicatorFactory(name="Age by Gender", dataset=dataset, groups=["gender"]) + category = profile_factoryboy.IndicatorCategoryFactory(name="Category", profile=profile) + subcategory = profile_factoryboy.IndicatorSubcategoryFactory(category=category, name="Subcategory") + + indicator_data = datasets_factoryboy.IndicatorDataFactory(indicator=indicator, geography=profile.geography_hierarchy.root_geography, data=indicator_data_items_data) + profile_indicator = profile_factoryboy.ProfileIndicatorFactory(label="Indicator", profile=profile, indicator=indicator, subcategory=subcategory) + datasets_factoryboy.GroupFactory(name="age group", dataset=dataset, subindicators=["15-19", "20-24"]), + datasets_factoryboy.GroupFactory(name="gender", dataset=dataset, subindicators=["M", "F"]), + + url = reverse("profile-geography-data", kwargs={"profile_id": profile.pk, "geography_code": profile.geography_hierarchy.root_geography.code}) + response = api_client.get(url, format='json') + groups = response.data.get('profile_data').get('Category').get('subcategories').get('Subcategory').get('indicators').get('Indicator').get('groups') + age_group = groups.get('age group') + assert age_group != expected_age_groups + + def test_profile_geography_data_ordering_is_correct_order(self, api_client, profile, indicator_data_items_data): + dataset = datasets_factoryboy.DatasetFactory(geography_hierarchy=profile.geography_hierarchy, groups=["age group", "gender"]) + indicator = datasets_factoryboy.IndicatorFactory(name="Age by Gender", dataset=dataset, groups=["gender"]) + category = profile_factoryboy.IndicatorCategoryFactory(name="Category", profile=profile) + subcategory = profile_factoryboy.IndicatorSubcategoryFactory(category=category, name="Subcategory") + + indicator_data = datasets_factoryboy.IndicatorDataFactory(indicator=indicator, geography=profile.geography_hierarchy.root_geography, data=indicator_data_items_data) + profile_indicator = profile_factoryboy.ProfileIndicatorFactory(label="Indicator", profile=profile, indicator=indicator, subcategory=subcategory) + datasets_factoryboy.GroupFactory(name="age group", dataset=dataset, subindicators=["15-19", "20-24"]), + datasets_factoryboy.GroupFactory(name="gender", dataset=dataset, subindicators=["M", "F"]), + + url = reverse("profile-geography-data", kwargs={"profile_id": profile.pk, "geography_code": profile.geography_hierarchy.root_geography.code}) + response = api_client.get(url, format='json') + groups = response.data.get('profile_data').get('Category').get('subcategories').get('Subcategory').get('indicators').get('Indicator').get('groups') + age_group = groups.get('age group') + wrong_order = OrderedDict([ + ('20-24',OrderedDict(F={"count": 2.2397})), + ('15-19',OrderedDict( + M={"count": 8.79722}, + F={"count": 9.62006}, + )), + ]) + + assert age_group != wrong_order + From 2559cf35fbb5e73d3554b0bd269d12ea893d5ff9 Mon Sep 17 00:00:00 2001 From: Adi Eyal Date: Tue, 29 Sep 2020 16:46:42 +0200 Subject: [PATCH 13/13] Fixed group ordering bug --- .../test_profile_indicator_sorter.py | 38 ++++++++++++++++++- .../serializers/test_subindicator_sorter.py | 35 +++++++++++++++++ tests/profile/test_views.py | 2 +- .../serializers/subindicator_sorter.py | 4 +- 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/tests/profile/serializers/test_profile_indicator_sorter.py b/tests/profile/serializers/test_profile_indicator_sorter.py index 81d53e27..74163ccc 100644 --- a/tests/profile/serializers/test_profile_indicator_sorter.py +++ b/tests/profile/serializers/test_profile_indicator_sorter.py @@ -94,18 +94,52 @@ def expected_subindicators(): @pytest.fixture def expected_groups(): + return [ + { + "group2": OrderedDict( + g2s2=OrderedDict(g1s3=6, g1s2=5, g1s1=4), + g2s1=OrderedDict(g1s3=3, g1s2=2, g1s1=1), + g2s3=OrderedDict(g1s3=9, g1s2=8, g1s1=7), + ), + }, + { + "group2": OrderedDict( + g2s2=OrderedDict(g1s3=60, g1s2=50, g1s1=40), + g2s1=OrderedDict(g1s3=30, g1s2=20, g1s1=10), + g2s3=OrderedDict(g1s3=90, g1s2=80, g1s1=70), + ) + } + ] + return { + "group2": [ + OrderedDict( + g2s2=OrderedDict(g1s3=6, g1s2=5, g1s1=4), + g2s1=OrderedDict(g1s3=3, g1s2=2, g1s1=1), + g2s3=OrderedDict(g1s3=9, g1s2=8, g1s1=7), + ), + OrderedDict( + g2s2=OrderedDict(g1s3=60, g1s2=50, g1s1=40), + g2s1=OrderedDict(g1s3=30, g1s2=20, g1s1=10), + g2s3=OrderedDict(g1s3=90, g1s2=80, g1s1=70), + ) + ] + } + + +@pytest.fixture +def alternative_expected_groups(): return [{ "group2": { - "g2s2": OrderedDict(g1s3=6, g1s2=5, g1s1=4), "g2s1": OrderedDict(g1s3=3, g1s2=2, g1s1=1), "g2s3": OrderedDict(g1s3=9, g1s2=8, g1s1=7), + "g2s2": OrderedDict(g1s3=6, g1s2=5, g1s1=4), } }, { "group2": { - "g2s2": OrderedDict(g1s3=60, g1s2=50, g1s1=40), "g2s1": OrderedDict(g1s3=30, g1s2=20, g1s1=10), "g2s3": OrderedDict(g1s3=90, g1s2=80, g1s1=70), + "g2s2": OrderedDict(g1s3=60, g1s2=50, g1s1=40), } }] diff --git a/tests/profile/serializers/test_subindicator_sorter.py b/tests/profile/serializers/test_subindicator_sorter.py index 9befd710..81bf9143 100644 --- a/tests/profile/serializers/test_subindicator_sorter.py +++ b/tests/profile/serializers/test_subindicator_sorter.py @@ -34,6 +34,17 @@ def sorter(): return sorter +@pytest.fixture +def alternative_sorter(): + group_orders = { + "group1": ["c", "a", "b"], + "group2": ["x", "z", "y"], + "group3": ["c3", "c1", "c2"], + } + sorter = SubindicatorSorter(group_orders) + + return sorter + class TestSubindicatorSorter: def test_subindicator_sort(self, sorter, subindicators): sort_order = ["b", "c", "a"] @@ -82,6 +93,7 @@ def test_group_sort(self, sorter, group_subindicators): sorted_group2 = sorted_groups["group2"] assert list(sorted_group1.keys()) == ["b", "c", "a"] + assert isinstance(sorted_group1, OrderedDict) sorted_group1_group3 = list(sorted_group1.values()) assert sorted_group1_group3[0] == OrderedDict(c2=200, c3=300, c1=100) @@ -89,11 +101,34 @@ def test_group_sort(self, sorter, group_subindicators): assert sorted_group1_group3[2] == OrderedDict(c2=20, c3=30, c1=10) assert list(sorted_group2.keys()) == ["z", "y", "x"] + assert isinstance(sorted_group2, OrderedDict) sorted_group2_group3 = list(sorted_group2.values()) assert sorted_group2_group3[0] == OrderedDict(c2=5000, c3=6000, c1=4000) assert sorted_group2_group3[1] == OrderedDict(c2=500, c3=600, c1=400) assert sorted_group2_group3[2] == OrderedDict(c2=50, c3=60, c1=40) + def test_alternative_group_sort(self, alternative_sorter, group_subindicators): + sorter = alternative_sorter + sorted_groups = sorter.sort_groups(group_subindicators, "group3") + sorted_group1 = sorted_groups["group1"] + sorted_group2 = sorted_groups["group2"] + + assert list(sorted_group1.keys()) == ["c", "a", "b"] + assert isinstance(sorted_group1, OrderedDict) + + sorted_group1_group3 = list(sorted_group1.values()) + assert sorted_group1_group3[0] == OrderedDict(c3=3000, c1=1000, c2=2000) + assert sorted_group1_group3[1] == OrderedDict(c3=30, c1=10, c2=20) + assert sorted_group1_group3[2] == OrderedDict(c3=300, c1=100, c2=200) + + assert list(sorted_group2.keys()) == ["x", "z", "y"] + assert isinstance(sorted_group2, OrderedDict) + + sorted_group2_group3 = list(sorted_group2.values()) + assert sorted_group2_group3[0] == OrderedDict(c3=60, c1=40, c2=50) + assert sorted_group2_group3[1] == OrderedDict(c3=6000, c1=4000, c2=5000) + assert sorted_group2_group3[2] == OrderedDict(c3=600, c1=400, c2=500) + def test_group_subset(self, sorter, group_subindicators): del group_subindicators["group1"] diff --git a/tests/profile/test_views.py b/tests/profile/test_views.py index b6322b0b..401c5802 100644 --- a/tests/profile/test_views.py +++ b/tests/profile/test_views.py @@ -69,7 +69,7 @@ def test_profile_geography_data_ordering_is_correct(self, api_client, profile, i response = api_client.get(url, format='json') groups = response.data.get('profile_data').get('Category').get('subcategories').get('Subcategory').get('indicators').get('Indicator').get('groups') age_group = groups.get('age group') - assert age_group != expected_age_groups + assert age_group == expected_age_groups def test_profile_geography_data_ordering_is_correct_order(self, api_client, profile, indicator_data_items_data): dataset = datasets_factoryboy.DatasetFactory(geography_hierarchy=profile.geography_hierarchy, groups=["age group", "gender"]) diff --git a/wazimap_ng/profile/serializers/subindicator_sorter.py b/wazimap_ng/profile/serializers/subindicator_sorter.py index 420cccce..486124e8 100644 --- a/wazimap_ng/profile/serializers/subindicator_sorter.py +++ b/wazimap_ng/profile/serializers/subindicator_sorter.py @@ -20,9 +20,9 @@ def sort_subindicators(self, subindicators, group): return sorted_dict def sort_groups(self, groups, primary_group): - new_dict = {} + new_dict = OrderedDict() for group, group_subindicators in groups.items(): - sorted_group_subindicators = dict(self.sort_subindicators(group_subindicators, group)) + sorted_group_subindicators = OrderedDict(self.sort_subindicators(group_subindicators, group)) for group_subindicator, subindicators in sorted_group_subindicators.items(): sorted_group_subindicators[group_subindicator] = self.sort_subindicators(subindicators, primary_group)