From e41d4009147d43f5e72df23167106bb326301acb Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Fri, 20 May 2022 10:21:30 -0400 Subject: [PATCH 01/24] v.dissolve: Compute attribute aggregate statistics In addition to geometry dissolving, compute aggregate statistics for the attribute values of dissolved features with v.db.univar. Requires v.db.univar JSON output. v.db.select with group is used to obtain unique values of the column the dissolving is based on. Add column and update now happens for every value, column, and statistics. --- scripts/v.dissolve/v.dissolve.py | 110 +++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index a22112cc187..86dd0052812 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -33,11 +33,34 @@ # %end # %option G_OPT_V_OUTPUT # %end +# %option G_OPT_DB_COLUMN +# % key: aggregate_column +# % description: Name of attribute columns to get aggregate statistics for +# % multiple: yes +# %end +# %option +# % key: aggregate_method +# % label: Aggregate statistics method +# % description: Default is all available basic statistics +# % multiple: yes +# %end +# %option G_OPT_DB_COLUMN +# % key: stats_column +# % description: New attribute column name for aggregate statistics results +# % description: Defaults to aggregate column name and statistics name +# % multiple: yes +# %end +# %rules +# % requires_all: aggregate_method,aggregate_column +# % requires_all: stats_column,aggregate_method,aggregate_column +# %end import os import atexit +import json import grass.script as grass +import grass.script as gs from grass.exceptions import CalledModuleError @@ -61,6 +84,28 @@ def main(): layer = options["layer"] column = options["column"] + aggregate_columns = options["aggregate_column"] + if aggregate_columns: + aggregate_columns = aggregate_columns.split(",") + else: + aggregate_columns = None + aggregate_methods = options["aggregate_method"] + if aggregate_methods: + aggregate_methods = aggregate_methods.split(",") + stats_columns = options["stats_column"] + if stats_columns: + stats_columns = stats_columns.split(",") + if len(stats_columns) != len(aggregate_columns) * len(aggregate_methods): + gs.fatal( + _( + "A column name is needed for each combination of aggregate_column " + "({num_columns}) and aggregate_method ({num_methods})" + ).format( + num_columns=len(aggregate_columns), + num_methods=len(aggregate_methods), + ) + ) + # setup temporary file tmp = str(os.getpid()) @@ -95,6 +140,10 @@ def main(): if coltype["type"] not in ("INTEGER", "SMALLINT", "CHARACTER", "TEXT"): grass.fatal(_("Key column must be of type integer or string")) + if coltype["type"] in ("CHARACTER", "TEXT"): + column_quote = True + else: + column_quote = False tmpfile = "%s_%s" % (output, tmp) @@ -110,6 +159,67 @@ def main(): type="area", layer=layer, ) + records = json.loads( + gs.read_command( + "v.db.select", + map=input, + columns=column, + group=column, + format="json", + ) + )["records"] + unique_values = [record[column] for record in records] + for value in unique_values: + for i, aggregate_column in enumerate(aggregate_columns): + if column_quote: + where = f"{column}='{value}'" + else: + where = f"{column}={value}" + stats = json.loads( + gs.read_command( + "v.db.univar", + map=input, + column=aggregate_column, + format="json", + where=where, + ) + )["statistics"] + if not aggregate_methods: + aggregate_methods = stats.keys() + if stats_columns: + current_stats_columns = stats_columns[ + i + * len(aggregate_methods) : (i + 1) + * len(aggregate_methods) + ] + else: + current_stats_columns = [ + f"{aggregate_column}_{method}" + for method in aggregate_methods + ] + for stats_column, key in zip( + current_stats_columns, aggregate_methods + ): + stats_value = stats[key] + # if stats_columns: + # stats_column = stats_columns[i * len(aggregate_methods) + j] + if key == "n": + stats_column_type = "INTEGER" + else: + stats_column_type = "DOUBLE" + gs.run_command( + "v.db.addcolumn", + map=output, + columns=f"{stats_column} {stats_column_type}", + ) + # TODO: Confirm that there is only one record in the table for a given attribute value after dissolve. + gs.run_command( + "v.db.update", + map=output, + column=stats_column, + value=stats_value, + where=where, + ) except CalledModuleError as e: grass.fatal( _( From 65c833d7015832d5f753ddc3ba20b58cb7be33a6 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 2 Jun 2022 19:01:36 -0400 Subject: [PATCH 02/24] Create columns only for the first value, support null (not clear how supported in the rest of the code), test --- scripts/v.dissolve/tests/conftest.py | 109 ++++++++++++++++++++ scripts/v.dissolve/tests/v_dissolve_test.py | 62 +++++++++++ scripts/v.dissolve/v.dissolve.py | 39 ++++--- 3 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 scripts/v.dissolve/tests/conftest.py create mode 100644 scripts/v.dissolve/tests/v_dissolve_test.py diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py new file mode 100644 index 00000000000..29645d95f0d --- /dev/null +++ b/scripts/v.dissolve/tests/conftest.py @@ -0,0 +1,109 @@ +"""Fixtures for v.dissolve tests""" + +from types import SimpleNamespace + +import pytest + +import grass.script as gs +import grass.script.setup as grass_setup + + +def updates_as_transaction(table, cat_column, column, column_quote, cats, values): + """Create SQL statement for categories and values for a given column""" + sql = ["BEGIN TRANSACTION"] + if column_quote: + quote = "'" + else: + quote = "" + for cat, value in zip(cats, values): + sql.append( + f"UPDATE {table} SET {column} = {quote}{value}{quote} " + f"WHERE {cat_column} = {cat};" + ) + sql.append("END TRANSACTION") + return "\n".join(sql) + + +def value_update_by_category(map_name, layer, column_name, cats, values): + """Update column value for multiple rows based on category""" + db_info = gs.vector_db(map_name)[layer] + table = db_info["table"] + database = db_info["database"] + driver = db_info["driver"] + cat_column = "cat" + column_type = gs.vector_columns(map_name, layer)[column_name] + column_quote = bool(column_type["type"] in ("CHARACTER", "TEXT")) + sql = updates_as_transaction( + table=table, + cat_column=cat_column, + column=column_name, + column_quote=column_quote, + cats=cats, + values=values, + ) + gs.write_command( + "db.execute", input="-", database=database, driver=driver, stdin=sql + ) + + +@pytest.fixture(scope="module") +def dataset(tmp_path_factory): + """Creates a session with a mapset which has vector with a float column""" + tmp_path = tmp_path_factory.mktemp("dataset") + location = "test" + point_map_name = "points" + map_name = "areas" + dissolve_column_name = "int_value" + value_column_name = "double_value" + str_column_name = "str_value" + + cats = [1, 2, 3, 4, 5, 6] + dissolve_values = [10, 10, 24, 5, 5, 5] + values = [100.78, 102.78, 109.78, 104.78, 103.78, 105.78] + str_values = ["apples", "oranges", "oranges", "plumbs", "oranges", "plumbs"] + num_points = len(cats) + + gs.core._create_location_xy(tmp_path, location) # pylint: disable=protected-access + with grass_setup.init(tmp_path / location): + gs.run_command("g.region", s=0, n=80, w=0, e=120, b=0, t=50, res=10, res3=10) + gs.run_command("v.random", output=point_map_name, npoints=num_points, seed=42) + gs.run_command("v.voronoi", input=point_map_name, output=map_name) + gs.run_command( + "v.db.addtable", + map=map_name, + columns=[ + f"{dissolve_column_name} integer", + f"{value_column_name} double precision", + f"{str_column_name} text", + ], + ) + value_update_by_category( + map_name=map_name, + layer=1, + column_name=dissolve_column_name, + cats=cats, + values=dissolve_values, + ) + value_update_by_category( + map_name=map_name, + layer=1, + column_name=value_column_name, + cats=cats, + values=values, + ) + value_update_by_category( + map_name=map_name, + layer=1, + column_name=str_column_name, + cats=cats, + values=str_values, + ) + yield SimpleNamespace( + vector_name=map_name, + dissolve_column_name=dissolve_column_name, + dissolve_values=dissolve_values, + value_column_name=value_column_name, + values=values, + str_column_name=str_column_name, + str_column_values=str_values, + ) diff --git a/scripts/v.dissolve/tests/v_dissolve_test.py b/scripts/v.dissolve/tests/v_dissolve_test.py new file mode 100644 index 00000000000..adee749533b --- /dev/null +++ b/scripts/v.dissolve/tests/v_dissolve_test.py @@ -0,0 +1,62 @@ +"""Test v.dissolve attribute aggregations""" + +import json + +import grass.script as gs + + +def test_aggregate_column(dataset): + """Check resulting types and values""" + dissolved_vector = "test1" + stats = ["sum", "n"] + stats_columns = ["value_sum", "value_n"] + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=dataset.value_column_name, + aggregate_method=stats, + stats_column=stats_columns, + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + + columns = gs.vector_columns(dissolved_vector) + assert len(columns) == 4 + for stats_column in stats_columns: + assert stats_column in columns + column_info = columns[stats_column] + if stats_column.endswith("_n"): + correct_type = "integer" + else: + correct_type = "double precision" + assert ( + columns[stats_column]["type"].lower() == correct_type + ), f"{stats_column} has a wrong type" + assert dataset.str_column_name in columns + column_info = columns[dataset.str_column_name] + assert column_info["type"].lower() == "character" + + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + format="json", + ) + )["records"] + ref_unique_values = set(dataset.str_column_values) + actual_values = [record[dataset.str_column_name] for record in records] + assert len(actual_values) == len(ref_unique_values) + assert set(actual_values) == ref_unique_values + + aggregate_n = [record["value_n"] for record in records] + assert sum(aggregate_n) == gs.vector_info(dataset.vector_name)["areas"] + assert sorted(aggregate_n) == [1, 2, 3] + aggregate_sum = [record["value_sum"] for record in records] + assert sorted(aggregate_sum) == [100.78, 210.56, 316.34] diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 86dd0052812..898cf2f261d 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -55,12 +55,16 @@ # % requires_all: stats_column,aggregate_method,aggregate_column # %end +"""Dissolve geometries and aggregate attribute values""" + import os import atexit import json import grass.script as grass -import grass.script as gs + +# To use new style of import without changing old code. +import grass.script as gs # pylint: disable=reimported from grass.exceptions import CalledModuleError @@ -140,10 +144,7 @@ def main(): if coltype["type"] not in ("INTEGER", "SMALLINT", "CHARACTER", "TEXT"): grass.fatal(_("Key column must be of type integer or string")) - if coltype["type"] in ("CHARACTER", "TEXT"): - column_quote = True - else: - column_quote = False + column_quote = bool(coltype["type"] in ("CHARACTER", "TEXT")) tmpfile = "%s_%s" % (output, tmp) @@ -169,9 +170,12 @@ def main(): ) )["records"] unique_values = [record[column] for record in records] + created_columns = set() for value in unique_values: for i, aggregate_column in enumerate(aggregate_columns): - if column_quote: + if value is None: + where = f"{column} IS NULL" + elif column_quote: where = f"{column}='{value}'" else: where = f"{column}={value}" @@ -203,16 +207,19 @@ def main(): stats_value = stats[key] # if stats_columns: # stats_column = stats_columns[i * len(aggregate_methods) + j] - if key == "n": - stats_column_type = "INTEGER" - else: - stats_column_type = "DOUBLE" - gs.run_command( - "v.db.addcolumn", - map=output, - columns=f"{stats_column} {stats_column_type}", - ) - # TODO: Confirm that there is only one record in the table for a given attribute value after dissolve. + if stats_column not in created_columns: + if key == "n": + stats_column_type = "INTEGER" + else: + stats_column_type = "DOUBLE" + gs.run_command( + "v.db.addcolumn", + map=output, + columns=f"{stats_column} {stats_column_type}", + ) + created_columns.add(stats_column) + # TODO: Confirm that there is only one record in the table + # for a given attribute value after dissolve. gs.run_command( "v.db.update", map=output, From 0b34030dc420544f2f3f08c3c850db0c53c41668 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 2 Jun 2022 23:34:06 -0400 Subject: [PATCH 03/24] More tests, fix optional aggregation and multiple column creation --- scripts/v.dissolve/tests/conftest.py | 28 ++-- .../tests/v_dissolve_aggregate_test.py | 147 ++++++++++++++++++ scripts/v.dissolve/tests/v_dissolve_test.py | 79 ++++++---- scripts/v.dissolve/v.dissolve.py | 139 +++++++++-------- 4 files changed, 287 insertions(+), 106 deletions(-) create mode 100644 scripts/v.dissolve/tests/v_dissolve_aggregate_test.py diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index 29645d95f0d..8f1e0dc74ba 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -53,13 +53,13 @@ def dataset(tmp_path_factory): location = "test" point_map_name = "points" map_name = "areas" - dissolve_column_name = "int_value" - value_column_name = "double_value" + int_column_name = "int_value" + float_column_name = "double_value" str_column_name = "str_value" cats = [1, 2, 3, 4, 5, 6] - dissolve_values = [10, 10, 24, 5, 5, 5] - values = [100.78, 102.78, 109.78, 104.78, 103.78, 105.78] + int_values = [10, 10, 10, 5, 24, 5] + float_values = [100.78, 102.78, 109.78, 104.78, 103.78, 105.78] str_values = ["apples", "oranges", "oranges", "plumbs", "oranges", "plumbs"] num_points = len(cats) @@ -72,24 +72,24 @@ def dataset(tmp_path_factory): "v.db.addtable", map=map_name, columns=[ - f"{dissolve_column_name} integer", - f"{value_column_name} double precision", + f"{int_column_name} integer", + f"{float_column_name} double precision", f"{str_column_name} text", ], ) value_update_by_category( map_name=map_name, layer=1, - column_name=dissolve_column_name, + column_name=int_column_name, cats=cats, - values=dissolve_values, + values=int_values, ) value_update_by_category( map_name=map_name, layer=1, - column_name=value_column_name, + column_name=float_column_name, cats=cats, - values=values, + values=float_values, ) value_update_by_category( map_name=map_name, @@ -100,10 +100,10 @@ def dataset(tmp_path_factory): ) yield SimpleNamespace( vector_name=map_name, - dissolve_column_name=dissolve_column_name, - dissolve_values=dissolve_values, - value_column_name=value_column_name, - values=values, + int_column_name=int_column_name, + int_values=int_values, + float_column_name=float_column_name, + float_values=float_values, str_column_name=str_column_name, str_column_values=str_values, ) diff --git a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py new file mode 100644 index 00000000000..5d6dd23624b --- /dev/null +++ b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py @@ -0,0 +1,147 @@ +"""Test v.dissolve attribute aggregations""" + +import json + +import pytest + +import grass.script as gs + + +@pytest.mark.parametrize( + "aggregate_methods", + [ + ["n"], + ["sum"], + ["range"], + ["min", "max", "mean", "variance"], + ["mean_abs", "stddev", "coeff_var"], + ], +) +def test_aggregate_methods(dataset, aggregate_methods): + """All aggregate methods are accepted and their columns generated""" + dissolved_vector = f"test_methods_{'_'.join(aggregate_methods)}" + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=dataset.float_column_name, + aggregate_method=aggregate_methods, + ) + columns = gs.vector_columns(dissolved_vector) + stats_columns = [ + f"{dataset.float_column_name}_{method}" for method in aggregate_methods + ] + assert sorted(columns.keys()) == sorted( + ["cat", dataset.str_column_name] + stats_columns + ) + + +def test_aggregate_two_columns(dataset): + """Aggregate stats for two columns are generated""" + dissolved_vector = "test_two_columns" + aggregate_methods = ["mean", "stddev"] + aggregate_columns = [dataset.float_column_name, dataset.int_column_name] + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=aggregate_columns, + aggregate_method=aggregate_methods, + ) + stats_columns = [ + f"{column}_{method}" + for method in aggregate_methods + for column in aggregate_columns + ] + columns = gs.vector_columns(dissolved_vector) + assert sorted(columns.keys()) == sorted( + ["cat", dataset.str_column_name] + stats_columns + ) + + +def test_aggregate_column_result(dataset): + """Check resulting types and values""" + dissolved_vector = "test_results" + stats = ["sum", "n"] + stats_columns = ["value_sum", "value_n"] + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=dataset.float_column_name, + aggregate_method=stats, + stats_column=stats_columns, + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + + columns = gs.vector_columns(dissolved_vector) + assert len(columns) == 4 + assert sorted(columns.keys()) == sorted( + ["cat", dataset.str_column_name] + stats_columns + ) + for stats_column in stats_columns: + assert stats_column in columns + column_info = columns[stats_column] + if stats_column.endswith("_n"): + correct_type = "integer" + else: + correct_type = "double precision" + assert ( + columns[stats_column]["type"].lower() == correct_type + ), f"{stats_column} has a wrong type" + assert dataset.str_column_name in columns + column_info = columns[dataset.str_column_name] + assert column_info["type"].lower() == "character" + + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + format="json", + ) + )["records"] + ref_unique_values = set(dataset.str_column_values) + actual_values = [record[dataset.str_column_name] for record in records] + assert len(actual_values) == len(ref_unique_values) + assert set(actual_values) == ref_unique_values + + aggregate_n = [record["value_n"] for record in records] + assert sum(aggregate_n) == gs.vector_info(dataset.vector_name)["areas"] + assert sorted(aggregate_n) == [1, 2, 3] + aggregate_sum = [record["value_sum"] for record in records] + assert sorted(aggregate_sum) == [ + dataset.float_values[0], + pytest.approx(dataset.float_values[3] + dataset.float_values[5]), + pytest.approx( + dataset.float_values[1] + dataset.float_values[2] + dataset.float_values[4] + ), + ] + + +def test_int_fails(dataset): + """An integer column fails with aggregates""" + dissolved_vector = "test_int" + stats = ["sum", "n"] + stats_columns = ["value_sum", "value_n"] + assert ( + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.int_column_name, + output=dissolved_vector, + aggregate_column=dataset.float_column_name, + aggregate_method=stats, + stats_column=stats_columns, + errors="status", + ) + != 0 + ) diff --git a/scripts/v.dissolve/tests/v_dissolve_test.py b/scripts/v.dissolve/tests/v_dissolve_test.py index adee749533b..92b50e3dfb8 100644 --- a/scripts/v.dissolve/tests/v_dissolve_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_test.py @@ -1,23 +1,49 @@ -"""Test v.dissolve attribute aggregations""" +"""Test v.dissolve geometry info""" import json import grass.script as gs -def test_aggregate_column(dataset): - """Check resulting types and values""" - dissolved_vector = "test1" - stats = ["sum", "n"] - stats_columns = ["value_sum", "value_n"] +def test_dissolve_int(dataset): + """Dissolving works on integer column""" + dissolved_vector = "test_int" + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.int_column_name, + output=dissolved_vector, + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["num_dblinks"] == 0 + # Reference values obtained by examining the result. + assert vector_info["north"] == 80 + assert vector_info["south"] == 0 + assert vector_info["east"] == 120 + assert vector_info["west"] == 0 + assert vector_info["nodes"] == 14 + assert vector_info["points"] == 0 + assert vector_info["lines"] == 0 + assert vector_info["boundaries"] == 16 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["islands"] == 1 + assert vector_info["primitives"] == 19 + assert vector_info["map3d"] == 0 + + +def test_dissolve_str(dataset): + """Dissolving works on string column and attributes are present""" + dissolved_vector = "test_str" gs.run_command( "v.dissolve", input=dataset.vector_name, column=dataset.str_column_name, output=dissolved_vector, - aggregate_column=dataset.value_column_name, - aggregate_method=stats, - stats_column=stats_columns, ) vector_info = gs.vector_info(dissolved_vector) @@ -26,20 +52,25 @@ def test_aggregate_column(dataset): assert vector_info["areas"] == 3 assert vector_info["num_dblinks"] == 1 assert vector_info["attribute_primary_key"] == "cat" + # Reference values obtained by examining the result. + assert vector_info["boundaries"] == 15 + assert vector_info["north"] == 80 + assert vector_info["south"] == 0 + assert vector_info["east"] == 120 + assert vector_info["west"] == 0 + assert vector_info["nodes"] == 13 + assert vector_info["points"] == 0 + assert vector_info["lines"] == 0 + assert vector_info["boundaries"] == 15 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["islands"] == 1 + assert vector_info["primitives"] == 18 + assert vector_info["map3d"] == 0 columns = gs.vector_columns(dissolved_vector) - assert len(columns) == 4 - for stats_column in stats_columns: - assert stats_column in columns - column_info = columns[stats_column] - if stats_column.endswith("_n"): - correct_type = "integer" - else: - correct_type = "double precision" - assert ( - columns[stats_column]["type"].lower() == correct_type - ), f"{stats_column} has a wrong type" - assert dataset.str_column_name in columns + assert len(columns) == 2 + assert sorted(columns.keys()) == sorted(["cat", dataset.str_column_name]) column_info = columns[dataset.str_column_name] assert column_info["type"].lower() == "character" @@ -54,9 +85,3 @@ def test_aggregate_column(dataset): actual_values = [record[dataset.str_column_name] for record in records] assert len(actual_values) == len(ref_unique_values) assert set(actual_values) == ref_unique_values - - aggregate_n = [record["value_n"] for record in records] - assert sum(aggregate_n) == gs.vector_info(dataset.vector_name)["areas"] - assert sorted(aggregate_n) == [1, 2, 3] - aggregate_sum = [record["value_sum"] for record in records] - assert sorted(aggregate_sum) == [100.78, 210.56, 316.34] diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 898cf2f261d..f6a408a0db0 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -144,7 +144,15 @@ def main(): if coltype["type"] not in ("INTEGER", "SMALLINT", "CHARACTER", "TEXT"): grass.fatal(_("Key column must be of type integer or string")) - column_quote = bool(coltype["type"] in ("CHARACTER", "TEXT")) + column_is_str = bool(coltype["type"] in ("CHARACTER", "TEXT")) + if aggregate_columns and not column_is_str: + grass.fatal( + _( + "Key column type must be string (text) " + "for aggregation method to work, not '{column_type}'" + ).format(column_type=coltype["type"]) + ) + column_quote = column_is_str tmpfile = "%s_%s" % (output, tmp) @@ -160,73 +168,74 @@ def main(): type="area", layer=layer, ) - records = json.loads( - gs.read_command( - "v.db.select", - map=input, - columns=column, - group=column, - format="json", - ) - )["records"] - unique_values = [record[column] for record in records] - created_columns = set() - for value in unique_values: - for i, aggregate_column in enumerate(aggregate_columns): - if value is None: - where = f"{column} IS NULL" - elif column_quote: - where = f"{column}='{value}'" - else: - where = f"{column}={value}" - stats = json.loads( - gs.read_command( - "v.db.univar", - map=input, - column=aggregate_column, - format="json", - where=where, - ) - )["statistics"] - if not aggregate_methods: - aggregate_methods = stats.keys() - if stats_columns: - current_stats_columns = stats_columns[ - i - * len(aggregate_methods) : (i + 1) - * len(aggregate_methods) - ] - else: - current_stats_columns = [ - f"{aggregate_column}_{method}" - for method in aggregate_methods - ] - for stats_column, key in zip( - current_stats_columns, aggregate_methods - ): - stats_value = stats[key] - # if stats_columns: - # stats_column = stats_columns[i * len(aggregate_methods) + j] - if stats_column not in created_columns: - if key == "n": - stats_column_type = "INTEGER" - else: - stats_column_type = "DOUBLE" + if aggregate_columns: + records = json.loads( + gs.read_command( + "v.db.select", + map=input, + columns=column, + group=column, + format="json", + ) + )["records"] + unique_values = [record[column] for record in records] + created_columns = set() + for value in unique_values: + for i, aggregate_column in enumerate(aggregate_columns): + if value is None: + where = f"{column} IS NULL" + elif column_quote: + where = f"{column}='{value}'" + else: + where = f"{column}={value}" + stats = json.loads( + gs.read_command( + "v.db.univar", + map=input, + column=aggregate_column, + format="json", + where=where, + ) + )["statistics"] + if not aggregate_methods: + aggregate_methods = stats.keys() + if stats_columns: + current_stats_columns = stats_columns[ + i + * len(aggregate_methods) : (i + 1) + * len(aggregate_methods) + ] + else: + current_stats_columns = [ + f"{aggregate_column}_{method}" + for method in aggregate_methods + ] + for stats_column, key in zip( + current_stats_columns, aggregate_methods + ): + stats_value = stats[key] + # if stats_columns: + # stats_column = stats_columns[i * len(aggregate_methods) + j] + if stats_column not in created_columns: + if key == "n": + stats_column_type = "INTEGER" + else: + stats_column_type = "DOUBLE" + gs.run_command( + "v.db.addcolumn", + map=output, + columns=f"{stats_column} {stats_column_type}", + ) + created_columns.add(stats_column) + # TODO: Confirm that there is only one record in the table + # for a given attribute value after dissolve. gs.run_command( - "v.db.addcolumn", + "v.db.update", map=output, - columns=f"{stats_column} {stats_column_type}", + column=stats_column, + value=stats_value, + where=where, ) - created_columns.add(stats_column) - # TODO: Confirm that there is only one record in the table - # for a given attribute value after dissolve. - gs.run_command( - "v.db.update", - map=output, - column=stats_column, - value=stats_value, - where=where, - ) except CalledModuleError as e: grass.fatal( _( From ea60dde08f32785054ab51ae3d15e18252dcb29c Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 7 Jun 2022 17:28:56 -0400 Subject: [PATCH 04/24] Create and update all columns at once --- scripts/v.dissolve/v.dissolve.py | 43 ++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index f6a408a0db0..6076cd82eb7 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -180,6 +180,8 @@ def main(): )["records"] unique_values = [record[column] for record in records] created_columns = set() + updates = [] + add_columns = [] for value in unique_values: for i, aggregate_column in enumerate(aggregate_columns): if value is None: @@ -221,21 +223,42 @@ def main(): stats_column_type = "INTEGER" else: stats_column_type = "DOUBLE" - gs.run_command( - "v.db.addcolumn", - map=output, - columns=f"{stats_column} {stats_column_type}", + add_columns.append( + f"{stats_column} {stats_column_type}" ) created_columns.add(stats_column) # TODO: Confirm that there is only one record in the table # for a given attribute value after dissolve. - gs.run_command( - "v.db.update", - map=output, - column=stats_column, - value=stats_value, - where=where, + updates.append( + { + "column": stats_column, + "value": stats_value, + "where": where, + } ) + gs.run_command( + "v.db.addcolumn", + map=output, + columns=",".join(add_columns), + ) + db_info = gs.vector_db(output)[1] + table = db_info["table"] + database = db_info["database"] + driver = db_info["driver"] + sql = ["BEGIN TRANSACTION"] + for update in updates: + sql.append( + f"UPDATE {table} SET {update['column']} = {update['value']} WHERE {update['where']};" + ) + sql.append("END TRANSACTION") + gs.write_command( + "db.execute", + input="-", + database=database, + driver=driver, + stdin="\n".join(sql), + ) + except CalledModuleError as e: grass.fatal( _( From 8ba12f62ca4ee7262e4f3b2fc7d20ffe35bf2904 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 7 Jun 2022 17:42:32 -0400 Subject: [PATCH 05/24] Create transaction in a function --- scripts/v.dissolve/v.dissolve.py | 33 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 6076cd82eb7..1d8e4de992e 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -8,7 +8,7 @@ # Converted to Python by Glynn Clements # PURPOSE: Dissolve common boundaries between areas with common cat # (frontend to v.extract -d) -# COPYRIGHT: (c) 2006-2014 Hamish Bowman, and the GRASS Development Team +# COPYRIGHT: (c) 2006-2022 Hamish Bowman, and the GRASS Development Team # This program is free software under the GNU General Public # License (>=v2). Read the file COPYING that comes with GRASS # for details. @@ -68,6 +68,18 @@ from grass.exceptions import CalledModuleError +def updates_to_sql(table, updates): + """Create SQL from a list of dicts with column, value, where""" + sql = ["BEGIN TRANSACTION"] + for update in updates: + sql.append( + f"UPDATE {table} SET {update['column']} = {update['value']} " + f"WHERE {update['where']};" + ) + sql.append("END TRANSACTION") + return "\n".join(sql) + + def cleanup(): nuldev = open(os.devnull, "w") grass.run_command( @@ -241,22 +253,15 @@ def main(): map=output, columns=",".join(add_columns), ) - db_info = gs.vector_db(output)[1] - table = db_info["table"] - database = db_info["database"] - driver = db_info["driver"] - sql = ["BEGIN TRANSACTION"] - for update in updates: - sql.append( - f"UPDATE {table} SET {update['column']} = {update['value']} WHERE {update['where']};" - ) - sql.append("END TRANSACTION") + output_layer = 1 + db_info = gs.vector_db(output)[output_layer] + sql = updates_to_sql(table=db_info["table"], updates=updates) gs.write_command( "db.execute", input="-", - database=database, - driver=driver, - stdin="\n".join(sql), + database=db_info["database"], + driver=db_info["driver"], + stdin=sql, ) except CalledModuleError as e: From 2b84604add15ccbed9c743f8e54715bfe41d2d7c Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 8 Jun 2022 16:13:25 -0400 Subject: [PATCH 06/24] Support direct SQL as a backend besides v.db.univar. Handle cleanup from the main function. Remove global variables. Use PID and node name for the temporary vector. --- .../tests/v_dissolve_aggregate_test.py | 128 +++++- scripts/v.dissolve/v.dissolve.py | 394 +++++++++++++----- 2 files changed, 409 insertions(+), 113 deletions(-) diff --git a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py index 5d6dd23624b..1b83f5d6fcd 100644 --- a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py @@ -1,6 +1,7 @@ """Test v.dissolve attribute aggregations""" import json +import statistics import pytest @@ -61,11 +62,15 @@ def test_aggregate_two_columns(dataset): ) -def test_aggregate_column_result(dataset): - """Check resulting types and values""" - dissolved_vector = "test_results" - stats = ["sum", "n"] - stats_columns = ["value_sum", "value_n"] +@pytest.mark.parametrize("backend", [None, "univar", "sql"]) +def test_aggregate_column_result(dataset, backend): + """Check resulting types and values of basic stats with different backends + + It assumes that the univar-like names are translated to SQLite names. + """ + dissolved_vector = f"test_results_{backend}" + stats = ["sum", "n", "min", "max", "mean"] + stats_columns = ["value_sum", "value_n", "value_min", "value_max", "value_mean"] gs.run_command( "v.dissolve", input=dataset.vector_name, @@ -74,6 +79,7 @@ def test_aggregate_column_result(dataset): aggregate_column=dataset.float_column_name, aggregate_method=stats, stats_column=stats_columns, + aggregate_backend=backend, ) vector_info = gs.vector_info(dissolved_vector) @@ -84,7 +90,7 @@ def test_aggregate_column_result(dataset): assert vector_info["attribute_primary_key"] == "cat" columns = gs.vector_columns(dissolved_vector) - assert len(columns) == 4 + assert len(columns) == len(stats_columns) + 2 assert sorted(columns.keys()) == sorted( ["cat", dataset.str_column_name] + stats_columns ) @@ -125,6 +131,116 @@ def test_aggregate_column_result(dataset): dataset.float_values[1] + dataset.float_values[2] + dataset.float_values[4] ), ] + aggregate_max = [record["value_max"] for record in records] + assert sorted(aggregate_max) == [ + dataset.float_values[0], + pytest.approx(max([dataset.float_values[3], dataset.float_values[5]])), + pytest.approx( + max( + [ + dataset.float_values[1], + dataset.float_values[2], + dataset.float_values[4], + ] + ) + ), + ] + aggregate_min = [record["value_min"] for record in records] + assert sorted(aggregate_min) == [ + dataset.float_values[0], + pytest.approx( + min( + [ + dataset.float_values[1], + dataset.float_values[2], + dataset.float_values[4], + ] + ) + ), + pytest.approx(min([dataset.float_values[3], dataset.float_values[5]])), + ] + aggregate_mean = [record["value_mean"] for record in records] + assert sorted(aggregate_mean) == [ + dataset.float_values[0], + pytest.approx( + statistics.mean([dataset.float_values[3], dataset.float_values[5]]) + ), + pytest.approx( + statistics.mean( + [ + dataset.float_values[1], + dataset.float_values[2], + dataset.float_values[4], + ] + ) + ), + ] + + +def test_sqlite_agg_accepted(dataset): + """Numeric SQLite aggregate function are accepted + + Additionally, it checks: + 1. generated column names + 2. types of columns + 3. aggregate counts + """ + dissolved_vector = "test_sqlite" + stats = ["avg", "count", "max", "min", "sum", "total"] + expected_stats_columns = [ + f"{dataset.float_column_name}_{method}" for method in stats + ] + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=dataset.float_column_name, + aggregate_method=stats, + aggregate_backend="sql", + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + + columns = gs.vector_columns(dissolved_vector) + assert len(columns) == len(expected_stats_columns) + 2 + assert sorted(columns.keys()) == sorted( + ["cat", dataset.str_column_name] + expected_stats_columns + ), "Unexpected autogenerated column names" + for method, stats_column in zip(stats, expected_stats_columns): + assert stats_column in columns + column_info = columns[stats_column] + if method == "count": + correct_type = "integer" + else: + correct_type = "double precision" + assert ( + columns[stats_column]["type"].lower() == correct_type + ), f"{stats_column} has a wrong type" + assert dataset.str_column_name in columns + column_info = columns[dataset.str_column_name] + assert column_info["type"].lower() == "character" + + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + format="json", + ) + )["records"] + ref_unique_values = set(dataset.str_column_values) + actual_values = [record[dataset.str_column_name] for record in records] + assert len(actual_values) == len(ref_unique_values) + assert set(actual_values) == ref_unique_values + + aggregate_n = [record[f"{dataset.float_column_name}_count"] for record in records] + assert sum(aggregate_n) == gs.vector_info(dataset.vector_name)["areas"] + assert sorted(aggregate_n) == [1, 2, 3] def test_int_fails(dataset): diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 1d8e4de992e..14ca41ca7aa 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -50,6 +50,14 @@ # % description: Defaults to aggregate column name and statistics name # % multiple: yes # %end +# %option +# % key: aggregate_backend +# % label: Aggregate statistics method +# % description: Default is all available basic statistics +# % multiple: no +# % required: no +# % options: univar,sql +# %end # %rules # % requires_all: aggregate_method,aggregate_column # % requires_all: stats_column,aggregate_method,aggregate_column @@ -68,6 +76,82 @@ from grass.exceptions import CalledModuleError +# Methods supported by v.db.univar by default. +UNIVAR_METHODS = [ + "n", + "min", + "max", + "range", + "mean", + "mean_abs", + "variance", + "stddev", + "coeff_var", + "sum", +] + +# Basic SQL aggregate function common between SQLite and PostgreSQL +# (and the SQL standard) using their proper names and order from +# their documentation. +# Notably, this does not include SQLite total which returns zero +# when all values are NULL. +STANDARD_SQL_FUNCTIONS = ["avg", "count", "max", "min", "sum"] + + +def get_methods_and_backend(methods, backend): + """Get methods and backed based on user-provided methods and backend""" + if methods: + if not backend: + in_univar = False + neither_in_sql_nor_univar = False + for method in methods: + if method not in STANDARD_SQL_FUNCTIONS: + if method in UNIVAR_METHODS: + in_univar = True + else: + neither_in_sql_nor_univar = True + # If all the non-basic functions are available in univar, use it. + if in_univar and not neither_in_sql_nor_univar: + backend = "univar" + elif backend == "sql": + methods = STANDARD_SQL_FUNCTIONS + elif backend == "univar": + methods = UNIVAR_METHODS + else: + # This is the default SQL functions but using the univar names (and order). + methods = ["n", "min", "max", "mean", "sum"] + backend = "sql" + if not backend: + backend = "sql" + return methods, backend + + +def modify_methods_for_backend(methods, backend): + """Modify list of methods to fit the backend if they do not + + This allows for support of the same method names for both backends. + It works both ways. + """ + new_methods = [] + if backend == "sql": + for method in methods: + if method == "n": + new_methods.append("count") + elif method == "mean": + new_methods.append("avg") + else: + new_methods.append(method) + elif backend == "univar": + for method in methods: + if method == "count": + new_methods.append("n") + elif method == "avg": + new_methods.append("mean") + else: + new_methods.append(method) + return new_methods + + def updates_to_sql(table, updates): """Create SQL from a list of dicts with column, value, where""" sql = ["BEGIN TRANSACTION"] @@ -80,50 +164,206 @@ def updates_to_sql(table, updates): return "\n".join(sql) -def cleanup(): +def update_columns(output, output_layer, updates, add_columns): + """Update attribute values based on a list of updates""" + if add_columns: + gs.run_command( + "v.db.addcolumn", + map=output, + columns=",".join(add_columns), + ) + db_info = gs.vector_db(output)[output_layer] + sql = updates_to_sql(table=db_info["table"], updates=updates) + gs.write_command( + "db.execute", + input="-", + database=db_info["database"], + driver=db_info["driver"], + stdin=sql, + ) + + +# TODO: Confirm that there is only one record in the table +# for a given attribute value after dissolve. + + +def aggregate_attributes_sql( + input_name, + column, + quote_column, + columns_to_aggregate, + methods, + result_columns, +): + """Aggregate values in selected columns grouped by column using SQL backend""" + select_columns = [ + f"{method}({agg_column})" + for method in methods + for agg_column in columns_to_aggregate + ] + column_types = [] + for unused_agg_column in columns_to_aggregate: + for method in methods: + if method == "count": + column_types.append("INTEGER") + else: + column_types.append("DOUBLE") + records = json.loads( + gs.read_command( + "v.db.select", + map=input_name, + columns=",".join([column] + select_columns), + group=column, + format="json", + ) + )["records"] + updates = [] + add_columns = [] + for stats_column, column_type in zip(result_columns, column_types): + add_columns.append(f"{stats_column} {column_type}") + for row in records: + value = row[column] + if value is None: + where = f"{column} IS NULL" + elif quote_column: + where = f"{column}='{value}'" + else: + where = f"{column}={value}" + for ( + stats_column, + key, + ) in zip(result_columns, select_columns): + updates.append( + { + "column": stats_column, + "value": row[key], + "where": where, + } + ) + return updates, add_columns + + +def aggregate_attributes_univar( + input_name, + column, + quote_column, + columns_to_aggregate, + methods, + result_columns, +): + """Aggregate values in selected columns grouped by column using v.db.univar""" + records = json.loads( + gs.read_command( + "v.db.select", + map=input_name, + columns=column, + group=column, + format="json", + ) + )["records"] + column_types = [] + for unused_agg_column in columns_to_aggregate: + for method in methods: + if method == "n": + column_types.append("INTEGER") + else: + column_types.append("DOUBLE") + add_columns = [] + for stats_column, column_type in zip(result_columns, column_types): + add_columns.append(f"{stats_column} {column_type}") + unique_values = [record[column] for record in records] + updates = [] + for value in unique_values: + if value is None: + where = f"{column} IS NULL" + elif quote_column: + where = f"{column}='{value}'" + else: + where = f"{column}={value}" + for i, aggregate_column in enumerate(columns_to_aggregate): + stats = json.loads( + gs.read_command( + "v.db.univar", + map=input_name, + column=aggregate_column, + format="json", + where=where, + ) + )["statistics"] + current_result_columns = result_columns[ + i * len(methods) : (i + 1) * len(methods) + ] + for stats_column, key in zip(current_result_columns, methods): + stats_value = stats[key] + updates.append( + { + "column": stats_column, + "value": stats_value, + "where": where, + } + ) + return updates, add_columns + + +def cleanup(name): + """Remove temporary vector silently""" nuldev = open(os.devnull, "w") grass.run_command( "g.remove", flags="f", type="vector", - name="%s_%s" % (output, tmp), + name=name, quiet=True, stderr=nuldev, ) -def main(): - global output, tmp +def option_as_list(options, name): + """Get value of an option as a list""" + option = options[name] + if option: + return option.split(",") + return [] + +def main(): + """Run the dissolve operation based on command line parameters""" + options, unused_flags = grass.parser() input = options["input"] output = options["output"] layer = options["layer"] column = options["column"] + aggregate_backend = options["aggregate_backend"] - aggregate_columns = options["aggregate_column"] - if aggregate_columns: - aggregate_columns = aggregate_columns.split(",") - else: - aggregate_columns = None - aggregate_methods = options["aggregate_method"] - if aggregate_methods: - aggregate_methods = aggregate_methods.split(",") - stats_columns = options["stats_column"] - if stats_columns: - stats_columns = stats_columns.split(",") - if len(stats_columns) != len(aggregate_columns) * len(aggregate_methods): + columns_to_aggregate = option_as_list(options, "aggregate_column") + user_aggregate_methods = option_as_list(options, "aggregate_method") + user_aggregate_methods, aggregate_backend = get_methods_and_backend( + user_aggregate_methods, aggregate_backend + ) + aggregate_methods = modify_methods_for_backend( + user_aggregate_methods, backend=aggregate_backend + ) + result_columns = options["stats_column"] + if result_columns: + result_columns = result_columns.split(",") + if len(result_columns) != len(columns_to_aggregate) * len( + user_aggregate_methods + ): gs.fatal( _( "A column name is needed for each combination of aggregate_column " "({num_columns}) and aggregate_method ({num_methods})" ).format( - num_columns=len(aggregate_columns), - num_methods=len(aggregate_methods), + num_columns=len(columns_to_aggregate), + num_methods=len(user_aggregate_methods), ) ) - - # setup temporary file - tmp = str(os.getpid()) + else: + result_columns = [ + f"{aggregate_column}_{method}" + for aggregate_column in columns_to_aggregate + for method in user_aggregate_methods + ] # does map exist? if not grass.find_file(input, element="vector")["file"]: @@ -157,16 +397,16 @@ def main(): if coltype["type"] not in ("INTEGER", "SMALLINT", "CHARACTER", "TEXT"): grass.fatal(_("Key column must be of type integer or string")) column_is_str = bool(coltype["type"] in ("CHARACTER", "TEXT")) - if aggregate_columns and not column_is_str: + if columns_to_aggregate and not column_is_str: grass.fatal( _( "Key column type must be string (text) " "for aggregation method to work, not '{column_type}'" ).format(column_type=coltype["type"]) ) - column_quote = column_is_str - tmpfile = "%s_%s" % (output, tmp) + tmpfile = gs.append_node_pid(output) + atexit.register(cleanup, tmpfile) try: grass.run_command( @@ -180,88 +420,30 @@ def main(): type="area", layer=layer, ) - if aggregate_columns: - records = json.loads( - gs.read_command( - "v.db.select", - map=input, - columns=column, - group=column, - format="json", + if columns_to_aggregate: + if aggregate_backend == "sql": + updates, add_columns = aggregate_attributes_sql( + input_name=input, + column=column, + quote_column=column_is_str, + columns_to_aggregate=columns_to_aggregate, + methods=aggregate_methods, + result_columns=result_columns, ) - )["records"] - unique_values = [record[column] for record in records] - created_columns = set() - updates = [] - add_columns = [] - for value in unique_values: - for i, aggregate_column in enumerate(aggregate_columns): - if value is None: - where = f"{column} IS NULL" - elif column_quote: - where = f"{column}='{value}'" - else: - where = f"{column}={value}" - stats = json.loads( - gs.read_command( - "v.db.univar", - map=input, - column=aggregate_column, - format="json", - where=where, - ) - )["statistics"] - if not aggregate_methods: - aggregate_methods = stats.keys() - if stats_columns: - current_stats_columns = stats_columns[ - i - * len(aggregate_methods) : (i + 1) - * len(aggregate_methods) - ] - else: - current_stats_columns = [ - f"{aggregate_column}_{method}" - for method in aggregate_methods - ] - for stats_column, key in zip( - current_stats_columns, aggregate_methods - ): - stats_value = stats[key] - # if stats_columns: - # stats_column = stats_columns[i * len(aggregate_methods) + j] - if stats_column not in created_columns: - if key == "n": - stats_column_type = "INTEGER" - else: - stats_column_type = "DOUBLE" - add_columns.append( - f"{stats_column} {stats_column_type}" - ) - created_columns.add(stats_column) - # TODO: Confirm that there is only one record in the table - # for a given attribute value after dissolve. - updates.append( - { - "column": stats_column, - "value": stats_value, - "where": where, - } - ) - gs.run_command( - "v.db.addcolumn", - map=output, - columns=",".join(add_columns), - ) - output_layer = 1 - db_info = gs.vector_db(output)[output_layer] - sql = updates_to_sql(table=db_info["table"], updates=updates) - gs.write_command( - "db.execute", - input="-", - database=db_info["database"], - driver=db_info["driver"], - stdin=sql, + else: + updates, add_columns = aggregate_attributes_univar( + input_name=input, + column=column, + quote_column=column_is_str, + columns_to_aggregate=columns_to_aggregate, + methods=aggregate_methods, + result_columns=result_columns, + ) + update_columns( + output=output, + output_layer=1, + updates=updates, + add_columns=add_columns, ) except CalledModuleError as e: @@ -279,6 +461,4 @@ def main(): if __name__ == "__main__": - options, flags = grass.parser() - atexit.register(cleanup) main() From 504595fc779f380d3adb48311e8ab74437d23ed4 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 8 Jun 2022 21:12:54 -0400 Subject: [PATCH 07/24] Functions for common aggregation functionality; shorter code. Simplify dev null in cleanup code. Modernize and Pylint generic error message. --- scripts/v.dissolve/v.dissolve.py | 93 ++++++++++++++++---------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 14ca41ca7aa..0da3e99eb3c 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -65,9 +65,9 @@ """Dissolve geometries and aggregate attribute values""" -import os import atexit import json +import subprocess import grass.script as grass @@ -183,6 +183,28 @@ def update_columns(output, output_layer, updates, add_columns): ) +def column_value_to_where(column, value, *, quote): + """Create SQL where clause without the where keyword for column and its value""" + if value is None: + return f"{column} IS NULL" + if quote: + return f"{column}='{value}'" + return f"{column}={value}" + + +def check_aggregate_methods_or_fatal(methods, backend): + if backend == "univar": + for method in methods: + if method not in UNIVAR_METHODS: + gs.fatal( + _( + "Method <{method}> is not available for backend <{backend}>" + ).format(method=method, backend=backend) + ) + # We don't have a list of available SQL functions. It is long for PostgreSQL + # and open for SQLite depending on its extensions. + + # TODO: Confirm that there is only one record in the table # for a given attribute value after dissolve. @@ -201,13 +223,9 @@ def aggregate_attributes_sql( for method in methods for agg_column in columns_to_aggregate ] - column_types = [] - for unused_agg_column in columns_to_aggregate: - for method in methods: - if method == "count": - column_types.append("INTEGER") - else: - column_types.append("DOUBLE") + column_types = [ + "INTEGER" if method == "count" else "DOUBLE" for method in methods + ] * len(columns_to_aggregate) records = json.loads( gs.read_command( "v.db.select", @@ -219,23 +237,17 @@ def aggregate_attributes_sql( )["records"] updates = [] add_columns = [] - for stats_column, column_type in zip(result_columns, column_types): - add_columns.append(f"{stats_column} {column_type}") + for result_column, column_type in zip(result_columns, column_types): + add_columns.append(f"{result_column} {column_type}") for row in records: - value = row[column] - if value is None: - where = f"{column} IS NULL" - elif quote_column: - where = f"{column}='{value}'" - else: - where = f"{column}={value}" + where = column_value_to_where(column, row[column], quote=quote_column) for ( - stats_column, + result_column, key, ) in zip(result_columns, select_columns): updates.append( { - "column": stats_column, + "column": result_column, "value": row[key], "where": where, } @@ -261,25 +273,16 @@ def aggregate_attributes_univar( format="json", ) )["records"] - column_types = [] - for unused_agg_column in columns_to_aggregate: - for method in methods: - if method == "n": - column_types.append("INTEGER") - else: - column_types.append("DOUBLE") + column_types = [ + "INTEGER" if method == "n" else "DOUBLE" for method in methods + ] * len(columns_to_aggregate) add_columns = [] - for stats_column, column_type in zip(result_columns, column_types): - add_columns.append(f"{stats_column} {column_type}") + for result_column, column_type in zip(result_columns, column_types): + add_columns.append(f"{result_column} {column_type}") unique_values = [record[column] for record in records] updates = [] for value in unique_values: - if value is None: - where = f"{column} IS NULL" - elif quote_column: - where = f"{column}='{value}'" - else: - where = f"{column}={value}" + where = column_value_to_where(column, value, quote=quote_column) for i, aggregate_column in enumerate(columns_to_aggregate): stats = json.loads( gs.read_command( @@ -293,12 +296,11 @@ def aggregate_attributes_univar( current_result_columns = result_columns[ i * len(methods) : (i + 1) * len(methods) ] - for stats_column, key in zip(current_result_columns, methods): - stats_value = stats[key] + for result_column, key in zip(current_result_columns, methods): updates.append( { - "column": stats_column, - "value": stats_value, + "column": result_column, + "value": stats[key], "where": where, } ) @@ -307,14 +309,13 @@ def aggregate_attributes_univar( def cleanup(name): """Remove temporary vector silently""" - nuldev = open(os.devnull, "w") grass.run_command( "g.remove", flags="f", type="vector", name=name, quiet=True, - stderr=nuldev, + stderr=subprocess.DEVNULL, ) @@ -343,9 +344,9 @@ def main(): aggregate_methods = modify_methods_for_backend( user_aggregate_methods, backend=aggregate_backend ) - result_columns = options["stats_column"] + check_aggregate_methods_or_fatal(aggregate_methods, backend=aggregate_backend) + result_columns = option_as_list(options, "stats_column") if result_columns: - result_columns = result_columns.split(",") if len(result_columns) != len(columns_to_aggregate) * len( user_aggregate_methods ): @@ -445,15 +446,13 @@ def main(): updates=updates, add_columns=add_columns, ) - - except CalledModuleError as e: + except CalledModuleError as error: grass.fatal( _( "Final extraction steps failed." " Check above error messages and" - " see following details:\n%s" - ) - % e + " see following details:\n{error}" + ).format(error=error) ) # write cmd history: From f7322ba09dbfa528342c758b1c505097e97ad628 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 8 Jun 2022 22:36:17 -0400 Subject: [PATCH 08/24] Test for dissolve/merge of areas without shared boundaries. --- scripts/v.dissolve/tests/conftest.py | 63 +++++++++++++++++++++ scripts/v.dissolve/tests/v_dissolve_test.py | 59 +++++++++++++++++-- scripts/v.dissolve/v.dissolve.py | 4 -- 3 files changed, 117 insertions(+), 9 deletions(-) diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index 8f1e0dc74ba..1ba7289516f 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -107,3 +107,66 @@ def dataset(tmp_path_factory): str_column_name=str_column_name, str_column_values=str_values, ) + + +@pytest.fixture(scope="module") +def discontinuous_dataset(tmp_path_factory): + """Creates a session with a mapset which has vector with a float column""" + tmp_path = tmp_path_factory.mktemp("discontinuous_dataset") + location = "test" + point_map_name = "points" + map_name = "areas" + int_column_name = "int_value" + float_column_name = "double_value" + str_column_name = "str_value" + + cats = [1, 2, 3, 4, 5, 6] + int_values = [10, 12, 10, 5, 24, 24] + float_values = [100.78, 102.78, 109.78, 104.78, 103.78, 105.78] + str_values = ["apples", "plumbs", "apples", "plumbs", "oranges", "oranges"] + num_points = len(cats) + + gs.core._create_location_xy(tmp_path, location) # pylint: disable=protected-access + with grass_setup.init(tmp_path / location): + gs.run_command("g.region", s=0, n=80, w=0, e=120, b=0, t=50, res=10, res3=10) + gs.run_command("v.random", output=point_map_name, npoints=num_points, seed=42) + gs.run_command("v.voronoi", input=point_map_name, output=map_name) + gs.run_command( + "v.db.addtable", + map=map_name, + columns=[ + f"{int_column_name} integer", + f"{float_column_name} double precision", + f"{str_column_name} text", + ], + ) + value_update_by_category( + map_name=map_name, + layer=1, + column_name=int_column_name, + cats=cats, + values=int_values, + ) + value_update_by_category( + map_name=map_name, + layer=1, + column_name=float_column_name, + cats=cats, + values=float_values, + ) + value_update_by_category( + map_name=map_name, + layer=1, + column_name=str_column_name, + cats=cats, + values=str_values, + ) + yield SimpleNamespace( + vector_name=map_name, + int_column_name=int_column_name, + int_values=int_values, + float_column_name=float_column_name, + float_values=float_values, + str_column_name=str_column_name, + str_column_values=str_values, + ) diff --git a/scripts/v.dissolve/tests/v_dissolve_test.py b/scripts/v.dissolve/tests/v_dissolve_test.py index 92b50e3dfb8..55001ff88b4 100644 --- a/scripts/v.dissolve/tests/v_dissolve_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_test.py @@ -29,8 +29,6 @@ def test_dissolve_int(dataset): assert vector_info["points"] == 0 assert vector_info["lines"] == 0 assert vector_info["boundaries"] == 16 - assert vector_info["centroids"] == 3 - assert vector_info["areas"] == 3 assert vector_info["islands"] == 1 assert vector_info["primitives"] == 19 assert vector_info["map3d"] == 0 @@ -53,7 +51,6 @@ def test_dissolve_str(dataset): assert vector_info["num_dblinks"] == 1 assert vector_info["attribute_primary_key"] == "cat" # Reference values obtained by examining the result. - assert vector_info["boundaries"] == 15 assert vector_info["north"] == 80 assert vector_info["south"] == 0 assert vector_info["east"] == 120 @@ -62,8 +59,6 @@ def test_dissolve_str(dataset): assert vector_info["points"] == 0 assert vector_info["lines"] == 0 assert vector_info["boundaries"] == 15 - assert vector_info["centroids"] == 3 - assert vector_info["areas"] == 3 assert vector_info["islands"] == 1 assert vector_info["primitives"] == 18 assert vector_info["map3d"] == 0 @@ -85,3 +80,57 @@ def test_dissolve_str(dataset): actual_values = [record[dataset.str_column_name] for record in records] assert len(actual_values) == len(ref_unique_values) assert set(actual_values) == ref_unique_values + + +def test_dissolve_discontinuous_str(discontinuous_dataset): + """Dissolving of discontinuous areas results in a single attribute record + + Even when the areas are discontinuous, there should be only one row + in the attribute table. + This behavior is assumed by the attribute aggregation functionality. + """ + dataset = discontinuous_dataset + dissolved_vector = "test_discontinuous_str" + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 5 + assert vector_info["areas"] == 5 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + # Reference values obtained by examining the result. + assert vector_info["north"] == 80 + assert vector_info["south"] == 0 + assert vector_info["east"] == 120 + assert vector_info["west"] == 0 + assert vector_info["nodes"] == 14 + assert vector_info["points"] == 0 + assert vector_info["lines"] == 0 + assert vector_info["boundaries"] == 18 + assert vector_info["islands"] == 1 + assert vector_info["primitives"] == 23 + assert vector_info["map3d"] == 0 + + columns = gs.vector_columns(dissolved_vector) + assert len(columns) == 2 + assert sorted(columns.keys()) == sorted(["cat", dataset.str_column_name]) + column_info = columns[dataset.str_column_name] + assert column_info["type"].lower() == "character" + + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + format="json", + ) + )["records"] + ref_unique_values = set(dataset.str_column_values) + actual_values = [record[dataset.str_column_name] for record in records] + assert len(actual_values) == len(ref_unique_values) + assert set(actual_values) == ref_unique_values diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 0da3e99eb3c..682ad8d6387 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -205,10 +205,6 @@ def check_aggregate_methods_or_fatal(methods, backend): # and open for SQLite depending on its extensions. -# TODO: Confirm that there is only one record in the table -# for a given attribute value after dissolve. - - def aggregate_attributes_sql( input_name, column, From 4da6a564a86fc993e588871e6ec1b8587fbd5a39 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 9 Jun 2022 11:31:36 -0400 Subject: [PATCH 09/24] Support any layer, not just 1 --- scripts/v.dissolve/tests/conftest.py | 76 ++++++++++++++++++ .../tests/v_dissolve_geometry_test.py | 59 ++++++++++++++ .../tests/v_dissolve_layers_test.py | 77 +++++++++++++++++++ scripts/v.dissolve/tests/v_dissolve_test.py | 54 ------------- scripts/v.dissolve/v.dissolve.py | 23 ++++-- 5 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 scripts/v.dissolve/tests/v_dissolve_geometry_test.py create mode 100644 scripts/v.dissolve/tests/v_dissolve_layers_test.py diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index 1ba7289516f..b74969999b1 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -170,3 +170,79 @@ def discontinuous_dataset(tmp_path_factory): str_column_name=str_column_name, str_column_values=str_values, ) + + +@pytest.fixture(scope="module") +def dataset_layer_2(tmp_path_factory): + """Creates a session with a mapset which has vector with a float column""" + tmp_path = tmp_path_factory.mktemp("dataset_layer_2") + location = "test" + point_map_name = "points" + point_map_name_layer_2 = "points2" + map_name = "areas" + int_column_name = "int_value" + float_column_name = "double_value" + str_column_name = "str_value" + + cats = [1, 2, 3, 4, 5, 6] + int_values = [10, 10, 10, 5, 24, 5] + float_values = [100.78, 102.78, 109.78, 104.78, 103.78, 105.78] + str_values = ["apples", "oranges", "oranges", "plumbs", "oranges", "plumbs"] + num_points = len(cats) + + layer = 2 + + gs.core._create_location_xy(tmp_path, location) # pylint: disable=protected-access + with grass_setup.init(tmp_path / location): + gs.run_command("g.region", s=0, n=80, w=0, e=120, b=0, t=50, res=10, res3=10) + gs.run_command("v.random", output=point_map_name, npoints=num_points, seed=42) + gs.run_command( + "v.category", + input=point_map_name, + layer=[1, layer], + output=point_map_name_layer_2, + option="transfer", + ) + gs.run_command( + "v.voronoi", input=point_map_name_layer_2, layer=layer, output=map_name + ) + gs.run_command( + "v.db.addtable", + map=map_name, + layer=layer, + columns=[ + f"{int_column_name} integer", + f"{float_column_name} double precision", + f"{str_column_name} text", + ], + ) + value_update_by_category( + map_name=map_name, + layer=layer, + column_name=int_column_name, + cats=cats, + values=int_values, + ) + value_update_by_category( + map_name=map_name, + layer=layer, + column_name=float_column_name, + cats=cats, + values=float_values, + ) + value_update_by_category( + map_name=map_name, + layer=layer, + column_name=str_column_name, + cats=cats, + values=str_values, + ) + yield SimpleNamespace( + vector_name=map_name, + int_column_name=int_column_name, + int_values=int_values, + float_column_name=float_column_name, + float_values=float_values, + str_column_name=str_column_name, + str_column_values=str_values, + ) diff --git a/scripts/v.dissolve/tests/v_dissolve_geometry_test.py b/scripts/v.dissolve/tests/v_dissolve_geometry_test.py new file mode 100644 index 00000000000..6989231cf17 --- /dev/null +++ b/scripts/v.dissolve/tests/v_dissolve_geometry_test.py @@ -0,0 +1,59 @@ +"""Test v.dissolve geometry info""" + +import json + +import grass.script as gs + + +def test_dissolve_discontinuous_str(discontinuous_dataset): + """Dissolving of discontinuous areas results in a single attribute record + + Even when the areas are discontinuous, there should be only one row + in the attribute table. + This behavior is assumed by the attribute aggregation functionality. + """ + dataset = discontinuous_dataset + dissolved_vector = "test_discontinuous_str" + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 5 + assert vector_info["areas"] == 5 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + # Reference values obtained by examining the result. + assert vector_info["north"] == 80 + assert vector_info["south"] == 0 + assert vector_info["east"] == 120 + assert vector_info["west"] == 0 + assert vector_info["nodes"] == 14 + assert vector_info["points"] == 0 + assert vector_info["lines"] == 0 + assert vector_info["boundaries"] == 18 + assert vector_info["islands"] == 1 + assert vector_info["primitives"] == 23 + assert vector_info["map3d"] == 0 + + columns = gs.vector_columns(dissolved_vector) + assert len(columns) == 2 + assert sorted(columns.keys()) == sorted(["cat", dataset.str_column_name]) + column_info = columns[dataset.str_column_name] + assert column_info["type"].lower() == "character" + + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + format="json", + ) + )["records"] + ref_unique_values = set(dataset.str_column_values) + actual_values = [record[dataset.str_column_name] for record in records] + assert len(actual_values) == len(ref_unique_values) + assert set(actual_values) == ref_unique_values diff --git a/scripts/v.dissolve/tests/v_dissolve_layers_test.py b/scripts/v.dissolve/tests/v_dissolve_layers_test.py new file mode 100644 index 00000000000..e86c32b3d77 --- /dev/null +++ b/scripts/v.dissolve/tests/v_dissolve_layers_test.py @@ -0,0 +1,77 @@ +"""More trickier tests of v.dissolve""" + +import json +import statistics + +import pytest + +import grass.script as gs + + +def test_layer_2(dataset_layer_2): + """Numeric SQLite aggregate function are accepted + + Additionally, it checks: + 1. generated column names + 2. types of columns + 3. aggregate counts + """ + dataset = dataset_layer_2 + dissolved_vector = "test_sqlite" + stats = ["avg", "count", "max", "min", "sum", "total"] + expected_stats_columns = [ + f"{dataset.float_column_name}_{method}" for method in stats + ] + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + layer=2, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=dataset.float_column_name, + aggregate_method=stats, + aggregate_backend="sql", + ) + + vector_info = gs.vector_info(dissolved_vector, layer=2) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + + columns = gs.vector_columns(dissolved_vector, layer=2) + assert len(columns) == len(expected_stats_columns) + 2 + assert sorted(columns.keys()) == sorted( + ["cat", dataset.str_column_name] + expected_stats_columns + ), "Unexpected autogenerated column names" + for method, stats_column in zip(stats, expected_stats_columns): + assert stats_column in columns + column_info = columns[stats_column] + if method == "count": + correct_type = "integer" + else: + correct_type = "double precision" + assert ( + columns[stats_column]["type"].lower() == correct_type + ), f"{stats_column} has a wrong type" + assert dataset.str_column_name in columns + column_info = columns[dataset.str_column_name] + assert column_info["type"].lower() == "character" + + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + layer=2, + format="json", + ) + )["records"] + ref_unique_values = set(dataset.str_column_values) + actual_values = [record[dataset.str_column_name] for record in records] + assert len(actual_values) == len(ref_unique_values) + assert set(actual_values) == ref_unique_values + + aggregate_n = [record[f"{dataset.float_column_name}_count"] for record in records] + assert sum(aggregate_n) == gs.vector_info(dataset.vector_name)["areas"] + assert sorted(aggregate_n) == [1, 2, 3] diff --git a/scripts/v.dissolve/tests/v_dissolve_test.py b/scripts/v.dissolve/tests/v_dissolve_test.py index 55001ff88b4..ded9aadeabe 100644 --- a/scripts/v.dissolve/tests/v_dissolve_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_test.py @@ -80,57 +80,3 @@ def test_dissolve_str(dataset): actual_values = [record[dataset.str_column_name] for record in records] assert len(actual_values) == len(ref_unique_values) assert set(actual_values) == ref_unique_values - - -def test_dissolve_discontinuous_str(discontinuous_dataset): - """Dissolving of discontinuous areas results in a single attribute record - - Even when the areas are discontinuous, there should be only one row - in the attribute table. - This behavior is assumed by the attribute aggregation functionality. - """ - dataset = discontinuous_dataset - dissolved_vector = "test_discontinuous_str" - gs.run_command( - "v.dissolve", - input=dataset.vector_name, - column=dataset.str_column_name, - output=dissolved_vector, - ) - - vector_info = gs.vector_info(dissolved_vector) - assert vector_info["level"] == 2 - assert vector_info["centroids"] == 5 - assert vector_info["areas"] == 5 - assert vector_info["num_dblinks"] == 1 - assert vector_info["attribute_primary_key"] == "cat" - # Reference values obtained by examining the result. - assert vector_info["north"] == 80 - assert vector_info["south"] == 0 - assert vector_info["east"] == 120 - assert vector_info["west"] == 0 - assert vector_info["nodes"] == 14 - assert vector_info["points"] == 0 - assert vector_info["lines"] == 0 - assert vector_info["boundaries"] == 18 - assert vector_info["islands"] == 1 - assert vector_info["primitives"] == 23 - assert vector_info["map3d"] == 0 - - columns = gs.vector_columns(dissolved_vector) - assert len(columns) == 2 - assert sorted(columns.keys()) == sorted(["cat", dataset.str_column_name]) - column_info = columns[dataset.str_column_name] - assert column_info["type"].lower() == "character" - - records = json.loads( - gs.read_command( - "v.db.select", - map=dissolved_vector, - format="json", - ) - )["records"] - ref_unique_values = set(dataset.str_column_values) - actual_values = [record[dataset.str_column_name] for record in records] - assert len(actual_values) == len(ref_unique_values) - assert set(actual_values) == ref_unique_values diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 682ad8d6387..35477120f28 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -164,15 +164,16 @@ def updates_to_sql(table, updates): return "\n".join(sql) -def update_columns(output, output_layer, updates, add_columns): +def update_columns(output_name, output_layer, updates, add_columns): """Update attribute values based on a list of updates""" if add_columns: gs.run_command( "v.db.addcolumn", - map=output, + map=output_name, + layer=output_layer, columns=",".join(add_columns), ) - db_info = gs.vector_db(output)[output_layer] + db_info = gs.vector_db(output_name)[int(output_layer)] sql = updates_to_sql(table=db_info["table"], updates=updates) gs.write_command( "db.execute", @@ -207,6 +208,7 @@ def check_aggregate_methods_or_fatal(methods, backend): def aggregate_attributes_sql( input_name, + input_layer, column, quote_column, columns_to_aggregate, @@ -226,6 +228,7 @@ def aggregate_attributes_sql( gs.read_command( "v.db.select", map=input_name, + layer=input_layer, columns=",".join([column] + select_columns), group=column, format="json", @@ -253,6 +256,7 @@ def aggregate_attributes_sql( def aggregate_attributes_univar( input_name, + input_layer, column, quote_column, columns_to_aggregate, @@ -264,6 +268,7 @@ def aggregate_attributes_univar( gs.read_command( "v.db.select", map=input_name, + layer=input_layer, columns=column, group=column, format="json", @@ -421,6 +426,7 @@ def main(): if aggregate_backend == "sql": updates, add_columns = aggregate_attributes_sql( input_name=input, + input_layer=layer, column=column, quote_column=column_is_str, columns_to_aggregate=columns_to_aggregate, @@ -430,6 +436,7 @@ def main(): else: updates, add_columns = aggregate_attributes_univar( input_name=input, + input_layer=layer, column=column, quote_column=column_is_str, columns_to_aggregate=columns_to_aggregate, @@ -437,17 +444,17 @@ def main(): result_columns=result_columns, ) update_columns( - output=output, - output_layer=1, + output_name=output, + output_layer=layer, updates=updates, add_columns=add_columns, ) except CalledModuleError as error: grass.fatal( _( - "Final extraction steps failed." - " Check above error messages and" - " see following details:\n{error}" + "A processing step failed." + " Check the above error messages and" + " see the following details:\n{error}" ).format(error=error) ) From d91a932ffac8cf3605c05304333fecf21171a9d2 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 9 Jun 2022 16:20:08 -0400 Subject: [PATCH 10/24] Check for valid methods more clear with ints than bools --- scripts/v.dissolve/v.dissolve.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 35477120f28..1ab2ae1408c 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -102,14 +102,14 @@ def get_methods_and_backend(methods, backend): """Get methods and backed based on user-provided methods and backend""" if methods: if not backend: - in_univar = False - neither_in_sql_nor_univar = False + in_univar = 0 + neither_in_sql_nor_univar = 0 for method in methods: if method not in STANDARD_SQL_FUNCTIONS: if method in UNIVAR_METHODS: - in_univar = True + in_univar += 1 else: - neither_in_sql_nor_univar = True + neither_in_sql_nor_univar += 1 # If all the non-basic functions are available in univar, use it. if in_univar and not neither_in_sql_nor_univar: backend = "univar" From ec6ff98cb42c3766cc8c437af3af0cc1785b16f0 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 9 Jun 2022 23:53:30 -0400 Subject: [PATCH 11/24] Notebook with examples and test --- scripts/v.dissolve/v_dissolve.ipynb | 279 ++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 scripts/v.dissolve/v_dissolve.ipynb diff --git a/scripts/v.dissolve/v_dissolve.ipynb b/scripts/v.dissolve/v_dissolve.ipynb new file mode 100644 index 00000000000..5adb48a596a --- /dev/null +++ b/scripts/v.dissolve/v_dissolve.ipynb @@ -0,0 +1,279 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# v.dissolve\n", + "\n", + "This notebook presents couple examples of _v.dissolve_ and examination of its outputs.\n", + "\n", + "## Setup\n", + "\n", + "We will be using the NC SPM sample location." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import subprocess\n", + "import sys\n", + "\n", + "# Ask GRASS GIS where its Python packages are.\n", + "sys.path.append(\n", + " subprocess.check_output([\"grass\", \"--config\", \"python_path\"], text=True).strip()\n", + ")\n", + "\n", + "# Import GRASS packages\n", + "import grass.script as gs\n", + "import grass.jupyter as gj\n", + "\n", + "# Start GRASS Session\n", + "gj.init(\"~/data/grassdata/nc_basic_spm_grass7/user1\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dissolve by Attribute\n", + "\n", + "We will use ZIP codes to create town boundaries by dissolving boundaries of ZIP code areas. Let's see the ZIP codes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "zipcodes = \"zipcodes\"\n", + "town_map = gj.Map()\n", + "town_map.d_vect(map=zipcodes)\n", + "town_map.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " We dissolve boudaries between ZIP codes which have the same town name which is in the NAME attribute." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "towns = \"towns_from_zipcodes\"\n", + "gs.run_command(\n", + " \"v.dissolve\",\n", + " input=zipcodes,\n", + " column=\"NAME\",\n", + " output=towns,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Color boudaries according to the primary key column called cat and display." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gs.run_command(\n", + " \"v.colors\",\n", + " map=towns,\n", + " use=\"attr\",\n", + " column=\"cat\",\n", + " color=\"wave\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "town_map.d_vect(map=towns)\n", + "town_map.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "town_map.d_vect(map=zipcodes, fill_color=\"none\")\n", + "town_map.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dissolve with Attribute Aggregation\n", + "\n", + "Now let's count number of ZIP codes in each town and compute total area as a sum of an existing column in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "towns_with_area = \"towns_with_area\"\n", + "gs.run_command(\n", + " \"v.dissolve\",\n", + " input=zipcodes,\n", + " column=\"NAME\",\n", + " output=towns_with_area,\n", + " aggregate_column=\"SHAPE_Area\",\n", + " aggregate_method=\"count,sum\",\n", + " stats_column=\"num_zip_codes,town_area\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Print the computed attributes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "table = json.loads(gs.read_command(\n", + " \"v.db.select\",\n", + " map=towns_with_area,\n", + " format=\"json\"\n", + "))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for row in table[\"records\"]:\n", + " print(f'{row[\"NAME\"]:<14} {row[\"num_zip_codes\"]:>2} {row[\"town_area\"]:>12.0f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now color the result using the total area:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gs.run_command(\n", + " \"v.colors\",\n", + " map=towns_with_area,\n", + " use=\"attr\",\n", + " column=\"town_area\",\n", + " color=\"plasma\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "town_map = gj.Map()\n", + "town_map.d_vect(map=towns_with_area)\n", + "town_map.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test\n", + "\n", + "For a small dataset, we can easily compute the same attribute values in Python. We do this assuming that all areas (polygons) with same value will be dissolved (merged) together possibly creating multipolygons." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "\n", + "# Get the original attribute data.\n", + "zip_table = json.loads(gs.read_command(\n", + " \"v.db.select\",\n", + " map=zipcodes,\n", + " format=\"json\"\n", + "))\n", + "# Restructure original data for easy lookup of area.\n", + "zip_records_by_town = defaultdict(list)\n", + "for row in zip_table[\"records\"]:\n", + " zip_records_by_town[row[\"NAME\"]].append(row[\"SHAPE_Area\"])\n", + "\n", + "# Check each row in the original table.\n", + "for row in table[\"records\"]:\n", + " town_name = row[\"NAME\"]\n", + " town_area = row[\"town_area\"]\n", + " town_zip_codes = row[\"num_zip_codes\"]\n", + " areas_by_zip = zip_records_by_town[town_name]\n", + " # Check number ZIP codes.\n", + " if len(areas_by_zip) != town_zip_codes:\n", + " raise RuntimeError(f'Incorrect number of zipcodes in town {row[\"NAME\"]}')\n", + " # Check total area.\n", + " if round(sum(areas_by_zip)) != round(town_area):\n", + " raise RuntimeError(f'Incorrect area for {row[\"NAME\"]}: {sum(areas_by_zip)} != {town_area}')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 1fee2b0707ca09228abd785fd02ca6b34aed65a5 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Mon, 13 Jun 2022 11:30:59 -0400 Subject: [PATCH 12/24] Do not generate all combinations when all options are provided, add documentation --- .../tests/v_dissolve_aggregate_test.py | 12 +- .../tests/v_dissolve_geometry_test.py | 2 +- .../tests/v_dissolve_layers_test.py | 5 +- scripts/v.dissolve/tests/v_dissolve_test.py | 2 +- scripts/v.dissolve/v.dissolve.html | 73 ++++++++++ scripts/v.dissolve/v.dissolve.py | 131 +++++++++++++----- 6 files changed, 176 insertions(+), 49 deletions(-) diff --git a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py index 1b83f5d6fcd..215776ee401 100644 --- a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py @@ -70,15 +70,16 @@ def test_aggregate_column_result(dataset, backend): """ dissolved_vector = f"test_results_{backend}" stats = ["sum", "n", "min", "max", "mean"] - stats_columns = ["value_sum", "value_n", "value_min", "value_max", "value_mean"] + stats_columns = [f"value_{method}" for method in stats] + aggregate_columns = [dataset.float_column_name] * len(stats) gs.run_command( "v.dissolve", input=dataset.vector_name, column=dataset.str_column_name, output=dissolved_vector, - aggregate_column=dataset.float_column_name, + aggregate_column=aggregate_columns, aggregate_method=stats, - stats_column=stats_columns, + result_column=stats_columns, aggregate_backend=backend, ) @@ -246,8 +247,6 @@ def test_sqlite_agg_accepted(dataset): def test_int_fails(dataset): """An integer column fails with aggregates""" dissolved_vector = "test_int" - stats = ["sum", "n"] - stats_columns = ["value_sum", "value_n"] assert ( gs.run_command( "v.dissolve", @@ -255,8 +254,7 @@ def test_int_fails(dataset): column=dataset.int_column_name, output=dissolved_vector, aggregate_column=dataset.float_column_name, - aggregate_method=stats, - stats_column=stats_columns, + aggregate_method="n", errors="status", ) != 0 diff --git a/scripts/v.dissolve/tests/v_dissolve_geometry_test.py b/scripts/v.dissolve/tests/v_dissolve_geometry_test.py index 6989231cf17..71c950a2141 100644 --- a/scripts/v.dissolve/tests/v_dissolve_geometry_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_geometry_test.py @@ -1,4 +1,4 @@ -"""Test v.dissolve geometry info""" +"""Test v.dissolve with more advanced geometry""" import json diff --git a/scripts/v.dissolve/tests/v_dissolve_layers_test.py b/scripts/v.dissolve/tests/v_dissolve_layers_test.py index e86c32b3d77..a13dc93315a 100644 --- a/scripts/v.dissolve/tests/v_dissolve_layers_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_layers_test.py @@ -1,9 +1,6 @@ -"""More trickier tests of v.dissolve""" +"""Tests of v.dissolve with layer other than 1""" import json -import statistics - -import pytest import grass.script as gs diff --git a/scripts/v.dissolve/tests/v_dissolve_test.py b/scripts/v.dissolve/tests/v_dissolve_test.py index ded9aadeabe..f5d579f5139 100644 --- a/scripts/v.dissolve/tests/v_dissolve_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_test.py @@ -1,4 +1,4 @@ -"""Test v.dissolve geometry info""" +"""Test v.dissolve geometry info and basic attributes""" import json diff --git a/scripts/v.dissolve/v.dissolve.html b/scripts/v.dissolve/v.dissolve.html index 5896b491aeb..ad1cdbd613f 100644 --- a/scripts/v.dissolve/v.dissolve.html +++ b/scripts/v.dissolve/v.dissolve.html @@ -7,6 +7,79 @@

DESCRIPTION

boundary dissolving. In this case the categories are not retained, only the values of the new key column. See the v.reclass help page for details. +

Merging behavior

+ +Multiple areas with the same category or the same attribute value +which are not adjacent are merged together into one entity +which consists of multiple areas, i.e., a multipolygon. + +

Attribute aggregation

+ +

+Attributes of merged areas can be aggregated using various aggregation methods +such as sum and mean. The specific methods available depend +on the backend used for aggregation. Two backends are available, +univar and sql. When univar +is used, the methods available are the ones which v.db.univar +uses by default, i.e., n, min, max, range, +mean, mean_abs, variance, stddev, +coef_var, and sum. +When the sql backend is used, the methods in turn depends on the SQL +database backend used for the attribute table of the input vector. +For SQLite, it is at least the following +build-in aggregate functions: +count, min, max, +avg, sum, and total. +For PostgreSQL, the list of +aggregate functions +is much longer and includes, e.g., count, min, max, +avg, sum, stddev, and variance. +The sql aggregate backend, regardless of the underlying database, +will typically perform significantly better than the univar backend. + +

+The backend is, by default, determined automatically based on the requested +methods. Specifically, the sql backend is used by default, +but when a method is not one of the SQLite build-in aggregate functions +and, at the same time, is available with the univar backend, +the univar backed is used. +The default behavior is intended for interactive use and testing. +For scripting and other automated usage, specifying the backend explicitly +is strongly recommended. + +

+For convince, certain methods, namely n, count, +mean, and avg, are converted to the name appropriate +for the selected backend. However, for scripting, specifying the appropriate +method (function) name for the backend is recommended because the conversion +is a heuristic which may change in the future. + +

+Type of the result column is determined based on the method selected. +For n and count, the type is INTEGER and for all other +methods, it is DOUBLE. Aggregate methods which don't produce other types +are not supported unless the value automatically converts to one of these. + +

+If only aggregate_column is provided, methods default to +n, min, max, mean, and sum. +If the univar backend is specified, all the available methods +for the univar backend are used. + +

+If the result_column is not provided, each method is applied to each +specified column producing result columns for all combinations. These result +columns have auto-generated names based on the aggregate column and method. +If the result_column is provided, each method is applied only once +to the matching column in the aggregate column list and the result will be +available under the name of the matching result column. In other words, number +of existing columns, methods, and new columns in aggregate_column, +aggregate_methods, and result_column needs to match and no +combinations are created on the fly. +For scripting, it is recommended to specify all resulting column names, +while for interactive use, automatically created combinations are expected +to be beneficial, especially for exploratory analysis. +

NOTES

GRASS defines a vector area as composite entity consisting of a set of diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 1ab2ae1408c..8a0567ac5fc 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -40,27 +40,27 @@ # %end # %option # % key: aggregate_method -# % label: Aggregate statistics method -# % description: Default is all available basic statistics +# % label: Aggregate statistics method (e.g., sum) +# % description: Default is all available basic statistics for a given backend # % multiple: yes # %end # %option G_OPT_DB_COLUMN -# % key: stats_column -# % description: New attribute column name for aggregate statistics results +# % key: result_column +# % label: New attribute column name for aggregate statistics results # % description: Defaults to aggregate column name and statistics name # % multiple: yes # %end # %option # % key: aggregate_backend -# % label: Aggregate statistics method -# % description: Default is all available basic statistics +# % label: Backend for attribute aggregation +# % description: Default is sql unless the methods are for univar # % multiple: no # % required: no -# % options: univar,sql +# % options: sql,univar # %end # %rules # % requires_all: aggregate_method,aggregate_column -# % requires_all: stats_column,aggregate_method,aggregate_column +# % requires_all: result_column,aggregate_method,aggregate_column # %end """Dissolve geometries and aggregate attribute values""" @@ -68,6 +68,7 @@ import atexit import json import subprocess +from collections import defaultdict import grass.script as grass @@ -194,6 +195,7 @@ def column_value_to_where(column, value, *, quote): def check_aggregate_methods_or_fatal(methods, backend): + """Check for known methods if possible or fail""" if backend == "univar": for method in methods: if method not in UNIVAR_METHODS: @@ -206,6 +208,61 @@ def check_aggregate_methods_or_fatal(methods, backend): # and open for SQLite depending on its extensions. +def match_columns_and_methods(columns, methods): + """Return all combinations of columns and methods""" + new_columns = [] + new_methods = [] + for column in columns: + for method in methods: + new_columns.append(column) + new_methods.append(method) + return new_columns, new_methods + + +def create_or_check_result_columns_or_fatal( + result_columns, columns_to_aggregate, methods +): + """Create result columns from input if not provided or check them""" + if result_columns: + if len(columns_to_aggregate) != len(methods): + gs.fatal( + _( + "When result columns are specified, the number of " + "aggregate columns ({columns_to_aggregate}) needs to be " + "the same as the number of methods ({methods})" + ).format( + columns_to_aggregate=len(columns_to_aggregate), + methods=len(methods), + ) + ) + if len(result_columns) != len(columns_to_aggregate): + gs.fatal( + _( + "The number of result columns ({result_columns}) needs to be " + "the same as the number of aggregate columns " + "({columns_to_aggregate})" + ).format( + result_columns=len(result_columns), + columns_to_aggregate=len(columns_to_aggregate), + ) + ) + if len(result_columns) != len(methods): + gs.fatal( + _( + "The number of result columns ({result_columns}) needs to be " + "the same as the number of aggregation methods ({methods})" + ).format( + result_columns=len(result_columns), + methods=len(methods), + ) + ) + return result_columns + return [ + f"{aggregate_column}_{method}" + for aggregate_column, method in zip(columns_to_aggregate, methods) + ] + + def aggregate_attributes_sql( input_name, input_layer, @@ -216,10 +273,14 @@ def aggregate_attributes_sql( result_columns, ): """Aggregate values in selected columns grouped by column using SQL backend""" + if len(columns_to_aggregate) != len(methods) != len(result_columns): + raise ValueError( + "Number of columns_to_aggregate, methods, and result_columns " + "must be the same" + ) select_columns = [ f"{method}({agg_column})" - for method in methods - for agg_column in columns_to_aggregate + for method, agg_column in zip(methods, columns_to_aggregate) ] column_types = [ "INTEGER" if method == "count" else "DOUBLE" for method in methods @@ -264,6 +325,11 @@ def aggregate_attributes_univar( result_columns, ): """Aggregate values in selected columns grouped by column using v.db.univar""" + if len(columns_to_aggregate) != len(methods) != len(result_columns): + raise ValueError( + "Number of columns_to_aggregate, methods, and result_columns " + "must be the same" + ) records = json.loads( gs.read_command( "v.db.select", @@ -274,6 +340,11 @@ def aggregate_attributes_univar( format="json", ) )["records"] + columns = defaultdict(list) + for agg_column, method, result in zip( + columns_to_aggregate, methods, result_columns + ): + columns[agg_column].append((method, result)) column_types = [ "INTEGER" if method == "n" else "DOUBLE" for method in methods ] * len(columns_to_aggregate) @@ -284,7 +355,8 @@ def aggregate_attributes_univar( updates = [] for value in unique_values: where = column_value_to_where(column, value, quote=quote_column) - for i, aggregate_column in enumerate(columns_to_aggregate): + # for i, aggregate_column in enumerate(columns_to_aggregate): + for aggregate_column, methods_results in columns.items(): stats = json.loads( gs.read_command( "v.db.univar", @@ -294,14 +366,11 @@ def aggregate_attributes_univar( where=where, ) )["statistics"] - current_result_columns = result_columns[ - i * len(methods) : (i + 1) * len(methods) - ] - for result_column, key in zip(current_result_columns, methods): + for method, result_column in methods_results: updates.append( { "column": result_column, - "value": stats[key], + "value": stats[method], "where": where, } ) @@ -342,30 +411,20 @@ def main(): user_aggregate_methods, aggregate_backend = get_methods_and_backend( user_aggregate_methods, aggregate_backend ) + result_columns = option_as_list(options, "result_column") + if not result_columns: + columns_to_aggregate, user_aggregate_methods = match_columns_and_methods( + columns_to_aggregate, user_aggregate_methods + ) aggregate_methods = modify_methods_for_backend( user_aggregate_methods, backend=aggregate_backend ) check_aggregate_methods_or_fatal(aggregate_methods, backend=aggregate_backend) - result_columns = option_as_list(options, "stats_column") - if result_columns: - if len(result_columns) != len(columns_to_aggregate) * len( - user_aggregate_methods - ): - gs.fatal( - _( - "A column name is needed for each combination of aggregate_column " - "({num_columns}) and aggregate_method ({num_methods})" - ).format( - num_columns=len(columns_to_aggregate), - num_methods=len(user_aggregate_methods), - ) - ) - else: - result_columns = [ - f"{aggregate_column}_{method}" - for aggregate_column in columns_to_aggregate - for method in user_aggregate_methods - ] + result_columns = create_or_check_result_columns_or_fatal( + result_columns=result_columns, + columns_to_aggregate=columns_to_aggregate, + methods=user_aggregate_methods, + ) # does map exist? if not grass.find_file(input, element="vector")["file"]: From f3f0f8469bd66e539baa3b0f533d5462c3b83f55 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Mon, 13 Jun 2022 17:23:46 -0400 Subject: [PATCH 13/24] Partially modernize the existing code by using gs alias instead of grass alias --- scripts/v.dissolve/v.dissolve.py | 58 ++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 8a0567ac5fc..e32b61090f4 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -6,6 +6,7 @@ # New Zealand # Markus Neteler for column support # Converted to Python by Glynn Clements +# Vaclav Petras (aggregate statistics) # PURPOSE: Dissolve common boundaries between areas with common cat # (frontend to v.extract -d) # COPYRIGHT: (c) 2006-2022 Hamish Bowman, and the GRASS Development Team @@ -35,7 +36,8 @@ # %end # %option G_OPT_DB_COLUMN # % key: aggregate_column -# % description: Name of attribute columns to get aggregate statistics for +# % label: Name of attribute columns to get aggregate statistics for +# % description: One column per method if result columns are specified # % multiple: yes # %end # %option @@ -70,10 +72,7 @@ import subprocess from collections import defaultdict -import grass.script as grass - -# To use new style of import without changing old code. -import grass.script as gs # pylint: disable=reimported +import grass.script as gs from grass.exceptions import CalledModuleError @@ -379,7 +378,7 @@ def aggregate_attributes_univar( def cleanup(name): """Remove temporary vector silently""" - grass.run_command( + gs.run_command( "g.remove", flags="f", type="vector", @@ -399,8 +398,8 @@ def option_as_list(options, name): def main(): """Run the dissolve operation based on command line parameters""" - options, unused_flags = grass.parser() - input = options["input"] + options, unused_flags = gs.parser() + input_vector = options["input"] output = options["output"] layer = options["layer"] column = options["column"] @@ -427,22 +426,27 @@ def main(): ) # does map exist? - if not grass.find_file(input, element="vector")["file"]: - grass.fatal(_("Vector map <%s> not found") % input) + if not gs.find_file(input_vector, element="vector")["file"]: + gs.fatal(_("Vector map <%s> not found") % input_vector) if not column: - grass.warning( + gs.warning( _( "No '%s' option specified. Dissolving based on category values from layer <%s>." ) % ("column", layer) ) - grass.run_command( - "v.extract", flags="d", input=input, output=output, type="area", layer=layer + gs.run_command( + "v.extract", + flags="d", + input=input_vector, + output=output, + type="area", + layer=layer, ) else: if int(layer) == -1: - grass.warning( + gs.warning( _( "Invalid layer number (%d). " "Parameter '%s' specified, assuming layer '1'." @@ -451,15 +455,15 @@ def main(): ) layer = "1" try: - coltype = grass.vector_columns(input, layer)[column] + coltype = gs.vector_columns(input_vector, layer)[column] except KeyError: - grass.fatal(_("Column <%s> not found") % column) + gs.fatal(_("Column <%s> not found") % column) if coltype["type"] not in ("INTEGER", "SMALLINT", "CHARACTER", "TEXT"): - grass.fatal(_("Key column must be of type integer or string")) + gs.fatal(_("Key column must be of type integer or string")) column_is_str = bool(coltype["type"] in ("CHARACTER", "TEXT")) if columns_to_aggregate and not column_is_str: - grass.fatal( + gs.fatal( _( "Key column type must be string (text) " "for aggregation method to work, not '{column_type}'" @@ -470,10 +474,14 @@ def main(): atexit.register(cleanup, tmpfile) try: - grass.run_command( - "v.reclass", input=input, output=tmpfile, layer=layer, column=column + gs.run_command( + "v.reclass", + input=input_vector, + output=tmpfile, + layer=layer, + column=column, ) - grass.run_command( + gs.run_command( "v.extract", flags="d", input=tmpfile, @@ -484,7 +492,7 @@ def main(): if columns_to_aggregate: if aggregate_backend == "sql": updates, add_columns = aggregate_attributes_sql( - input_name=input, + input_name=input_vector, input_layer=layer, column=column, quote_column=column_is_str, @@ -494,7 +502,7 @@ def main(): ) else: updates, add_columns = aggregate_attributes_univar( - input_name=input, + input_name=input_vector, input_layer=layer, column=column, quote_column=column_is_str, @@ -509,7 +517,7 @@ def main(): add_columns=add_columns, ) except CalledModuleError as error: - grass.fatal( + gs.fatal( _( "A processing step failed." " Check the above error messages and" @@ -518,7 +526,7 @@ def main(): ) # write cmd history: - grass.vector_history(output) + gs.vector_history(output) if __name__ == "__main__": From 415e666a896c95dedbbb5928f362a94194aba0cb Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Mon, 13 Jun 2022 17:34:48 -0400 Subject: [PATCH 14/24] Image to doc, generated in notebook --- scripts/v.dissolve/v.dissolve.html | 16 +++- scripts/v.dissolve/v_dissolve.ipynb | 84 +++++++++++++++------ scripts/v.dissolve/v_dissolve_towns.png | Bin 0 -> 24161 bytes scripts/v.dissolve/v_dissolve_zipcodes.png | Bin 0 -> 30257 bytes 4 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 scripts/v.dissolve/v_dissolve_towns.png create mode 100644 scripts/v.dissolve/v_dissolve_zipcodes.png diff --git a/scripts/v.dissolve/v.dissolve.html b/scripts/v.dissolve/v.dissolve.html index ad1cdbd613f..f378f11c063 100644 --- a/scripts/v.dissolve/v.dissolve.html +++ b/scripts/v.dissolve/v.dissolve.html @@ -7,6 +7,15 @@

DESCRIPTION

boundary dissolving. In this case the categories are not retained, only the values of the new key column. See the v.reclass help page for details. +
+ + +

+ Figure: Areas with the same attribute value (first image) are + merged into into one (second image) +

+
+

Merging behavior

Multiple areas with the same category or the same attribute value @@ -141,6 +150,7 @@

SEE ALSO

AUTHORS

-module: M. Hamish Bowman, Dept. Marine Science, Otago University, New Zealand
-Markus Neteler for column support
-help page: Trevor Wiens +M. Hamish Bowman, Dept. Marine Science, Otago University, New Zealand (module)
+Markus Neteler (column support)
+Trevor Wiens (help page)
+Vaclav Petras, NC State University, Center for Geospatial Analytics (aggregate statistics) diff --git a/scripts/v.dissolve/v_dissolve.ipynb b/scripts/v.dissolve/v_dissolve.ipynb index 5adb48a596a..f90907a704d 100644 --- a/scripts/v.dissolve/v_dissolve.ipynb +++ b/scripts/v.dissolve/v_dissolve.ipynb @@ -93,13 +93,7 @@ "metadata": {}, "outputs": [], "source": [ - "gs.run_command(\n", - " \"v.colors\",\n", - " map=towns,\n", - " use=\"attr\",\n", - " column=\"cat\",\n", - " color=\"wave\"\n", - ")" + "gs.run_command(\"v.colors\", map=towns, use=\"attr\", column=\"cat\", color=\"wave\")" ] }, { @@ -143,9 +137,9 @@ " input=zipcodes,\n", " column=\"NAME\",\n", " output=towns_with_area,\n", - " aggregate_column=\"SHAPE_Area\",\n", + " aggregate_column=\"SHAPE_Area,SHAPE_Area\",\n", " aggregate_method=\"count,sum\",\n", - " stats_column=\"num_zip_codes,town_area\"\n", + " result_column=\"num_zip_codes,town_area\",\n", ")" ] }, @@ -162,11 +156,7 @@ "metadata": {}, "outputs": [], "source": [ - "table = json.loads(gs.read_command(\n", - " \"v.db.select\",\n", - " map=towns_with_area,\n", - " format=\"json\"\n", - "))" + "table = json.loads(gs.read_command(\"v.db.select\", map=towns_with_area, format=\"json\"))" ] }, { @@ -193,11 +183,7 @@ "outputs": [], "source": [ "gs.run_command(\n", - " \"v.colors\",\n", - " map=towns_with_area,\n", - " use=\"attr\",\n", - " column=\"town_area\",\n", - " color=\"plasma\"\n", + " \"v.colors\", map=towns_with_area, use=\"attr\", column=\"town_area\", color=\"plasma\"\n", ")" ] }, @@ -212,6 +198,55 @@ "town_map.show()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Images for Documentation\n", + "\n", + "Here, we use some of the data created above to create images for documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "zip_map = gj.Map()\n", + "zip_map.d_vect(map=towns, flags=\"s\")\n", + "zip_map.d_vect(map=zipcodes, color=\"#222222\", width=2, type=\"boundary\")\n", + "zip_map.d_legend_vect()\n", + "zip_map.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "town_map = gj.Map()\n", + "town_map.d_vect(map=towns, flags=\"s\")\n", + "town_map.d_vect(map=towns_with_area, color=\"#222222\", width=2, type=\"boundary\")\n", + "town_map.d_legend_vect()\n", + "town_map.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# This cell requires pngquant and optipng.\n", + "zip_map.save(\"v_dissolve_zipcodes.png\")\n", + "town_map.save(\"v_dissolve_towns.png\")\n", + "for filename in [\"v_dissolve_zipcodes.png\", \"v_dissolve_towns.png\"]:\n", + " !pngquant --ext \".png\" -f {filename}\n", + " !optipng -o7 {filename}" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -230,11 +265,7 @@ "from collections import defaultdict\n", "\n", "# Get the original attribute data.\n", - "zip_table = json.loads(gs.read_command(\n", - " \"v.db.select\",\n", - " map=zipcodes,\n", - " format=\"json\"\n", - "))\n", + "zip_table = json.loads(gs.read_command(\"v.db.select\", map=zipcodes, format=\"json\"))\n", "# Restructure original data for easy lookup of area.\n", "zip_records_by_town = defaultdict(list)\n", "for row in zip_table[\"records\"]:\n", @@ -251,7 +282,10 @@ " raise RuntimeError(f'Incorrect number of zipcodes in town {row[\"NAME\"]}')\n", " # Check total area.\n", " if round(sum(areas_by_zip)) != round(town_area):\n", - " raise RuntimeError(f'Incorrect area for {row[\"NAME\"]}: {sum(areas_by_zip)} != {town_area}')" + " raise RuntimeError(\n", + " f'Incorrect area for {row[\"NAME\"]}: {sum(areas_by_zip)} != {town_area}'\n", + " )\n", + "print(\"No exceptions. Test passed.\")" ] } ], diff --git a/scripts/v.dissolve/v_dissolve_towns.png b/scripts/v.dissolve/v_dissolve_towns.png new file mode 100644 index 0000000000000000000000000000000000000000..52df4e7d67e0ca595b99d994a8befd1f8b9cf4f9 GIT binary patch literal 24161 zcmV)SK(fDyP)a5uIpZQDBjPx6tEmAbA|xUvB_<*$Q&syUCMzi-E+{BFFCjNV zL$EI>BPA~;Ff&&vBqKHwT0i!!Tl2A7(uznplIe#owF=s$4YaWx}a5zvOYAiixc^;^F zM=dTaHZ&wH3V?t`AX+3xH3yE4Voo3)V1P$3C?QipejTtxJR%@NRE1(6T0$nQTQZ3q zz>sMkXE8G*Juoq1FqBa*NFrmUcOFsgQBhO!i#j?Te5;5|E}&L1n{XZuZ*Rj#M=~r; zQ~6UWWIP{HYMKL@TOWElhdJ<3QBy20(n?AaVPQrhIyG-Iy&Q-Hnwnn}U~g0)xnDFe zmr^P|D6<@a9*}7?WyMPwOm;OkA9-3pl{zeUMIM`Q9d4F)A6ijCBH~U?P*Uz{5NRBS zy4!9z`BPI-BSI`lE755-VJ1ePKR-0UW03}tJRmhVdo?y~)&ZlVNy|wcw176wXmky8 zP*UqwA9BKDHAcQhK&C(yQ&TaeR~^ENkREDLX&^>VAG1S4%V##H9eH|OAJR5!h8?|N ze;^&ag&=HEFH@NwrFViIwoKSeR2Wr+9%3P3N-Rl$L$O02g<>*YuukJnGqhdSYd3`* zyB(~1X(3T=mL3pGV)ZBJ`fjy7oTzz{Mumvi8Os0wTJ}jqK~#9!?43_f8eJE_t0EPUqA32E zS)L8i+y^?~Q(yWtwStC+IHW-m>%2`96GBWHc67sv#AP=mZ_;HyL6?4jhb3&34IjXS zkoX0lDmN5!E>7U%Lb{Aonp1DerP`JEQ7&th z^3N>*JF10roBNnkm)>r;DpR#Qs21`FGfQq)k4`lsS7{CG?pOE-k%w5IPHpw-)QwtO zV0TR7Lqyt{h4PiRZ&bD1s20-3B2>JCz|;z1)I#oKg_?ITYG4O&Ea0O=?qY+>0mUjq zz9<1$SMXsXH%!6SfL>j$k*s*3s%63FiQFs-zYQo>3GyN(S3tFpyNkk7-=RwTNoc@Q z$t9^;)2J3Q*rKpPO4n+@%7{&J$skp0_9Z@6r0+%H)fT|Jn)h-Gj>TJKR14{OQFx_| zH)tk)fJ6wOT!Mtyf~-)rT&NZ<3cGWvUars>wYCOSCU32BS%ToXT=3Cf!$kT^K!X##ds3s;4ACl}JSrKRu1-E}pkoLyuT0Y^ug;!CGsq={(&~gMa7nr!aQzFb~>e_Z;9199Y42- z#nw3S3VwDfu`;3|X34CM2geF##Bt%}1iUkJN9XZ6o^4RO<}o9V2~+Uqaj|?*6^-M( z5X6i)Hq3N5Vn3X@g`(c;N||Or+szAInFFWC=+$APYnsA~AIz`TsvSODd26LDfI0Ww4*1N( z%#5?23qgjK5wUA(Y9?t7&%T{%oo#=Ov-VM$C0ElD`&O{cwTjC`v$jfU4ewfgxTcvM zNiu9qF|&+Fv18maD=Uvq+${|E;A+nqF_ldL;AD+?E3HSDjRuW5beP)0$(nIwX#|@) zZTpDoWQ_$UYsLwW?jy=LS@SG$8#tKt^kj`^1>G7Ou~H%*s9jUM`(a#gG{rjSAtOcz zPS%W#SlY#Y3Lk;%f@MS-9?&;GR!uC0RC&pW>cJk*D_kRvgIEd)iTn^x(2VGGU`8~( zSV=PaZq~i~a@8^B6?DtgVkvYr#+O))_5)ZaQ7w})4e_iG<~ZqQwG~_7!Yky>Dprty zdk(51hs3E`_Qu33Vm7hBK5c=J$R+0rn+F8vEZ&D?HZ04QnHQ_vl~{M8mRf8%a2U{} zg_g)K>KD$5XK@)bqRE)Dlbpyi>J9R=kjFaSWa)6UQ>5Lgl@5jQn<^$_aa0E?dK{5Y z(4d8CnFtpgcM29ce!>>}Tn`REn1u;MGQG#bQ&c0PiSuhGD`b~oTxn8TSZK44SH7ij{ochw@k01|se z2UMP@)^lLNs}27VZ)VByL?R)<-(bae$^}+PB_$w7zi5O?Y8!*mgFu+dvq7S7KW^^q z3J>xLq+^jc5>WpN^Y~IZV8%{@0K0o?gd(Ak4`Ta5c$=wn_*UqE|A0fYGElF>9G<)2 zfE5~{b{bXG5leip{X%!}XjUHT>!;wd;Cmf8%fZ*trb^+xZHmw>;av!3`Kq(hRIIbV z@%A`}RtBYy3CqoSU;=(piv7G;>hqov~RC|9q@TDzY1U1DRq_* z@ydr0;ezeH#iENyE)V9*K$uuGYN1-A;uZYWerEm)I9u2FO=~R&?j9P?yt;SO zCmWFGQ>-BT+cfJ;RSUNdjm81*`xonw*Bc+$1tLweRJCk)O^w0b2Iqr2%w4j_-J-C0 ztVklaVo|mJ2X1`VhI%y#JNt~-~7T>m!EWyzLVhIwQn?kJhI!6}g*;wRtlJwjt^BSqxcbheRU9 zLXx4R(%5q_#485`_Jv_{vav&}rqo@vau17jc>9l{oo}if*O^s-Pkuoc{OdK1Dww3Z(EUk2T9v>RtzUJcthk<4$R2thkt>P68IvXb8Z8Ul|8v?(K73<{0AArUsfT0Jrl46E^ zlsbhCc->gAuN?{o!}bKVLp|}f3X63pd~jiqS$Q1#(}}77vv)l&ZKQvg?zigttG2aj zY9W`;LA1ObGmciHD44{m>jZZU7_%buOOSx%pac&=mI3u*V96S3Z+i)af>NZI!`}3q zLv!*E=%sk_6nl{3zVGCRF_}pwlWCGM^PIY?E8Fzx=jZcz-o_H;GD); zj+nF&bD*5WVi{=Dr&;Q*awaOlD+^7gqd~Au=DB_?quCbAgI;E7dXop5c*lAiDf~EEhDjB_{{n?TGOkXR5%h^K0i>BBh+$xBI(;` zHCH(+mEqMw@)6e@Vr?%ijZV%j938DV%cCLHx6wZV&1xSK7UA2S5QzH0l>LQUl>IFN z9_sV~yb9=qv4zU;Ds3v6L##ny%w~q6X$qD2dWtLPaM_Fd5q?IupiyI$(^Q%nR!O%N z$&=JrHnB2R&n1@bULoqy-XeYs>(S4_R9NM-mFC{37O^Z@R`F^jKre87fCiwf>=oeybFtV@}?K^Okt z_>z2i=ljXKk;IQ`t?$t3l^+k=8tchNHLzVtVvQ098sCdpEOzemc}W*xp@BlTC#(ts zmL2GTm%q8fxV(&K604V%w|ex@DPkpEaIe3oqZg)Er$I06#d*~vd_==5lid}?FaRBU zkuslg3eWyhc8_b-U1G%Idc|==C#28?7lyEQ{W1qQIfhg8()T`MsDt^riP`D-(@L`H zXmF4M#zDM{=>YE%Dp=a=%J1f+h{Xd~3rDON1#t4^r4(@VLV)y(94^1x4IolW;u_gIc|#=*y>is(iEY8O_RripgZ|bcCe&TPZL&RT8iA z63Zs@{*{Awk|8Su2gXBas~KW-;m_0tkL#Zsek~+tsFY0RZpL%YOL@FGm7hN&;CX3Y z-H_W2nTKFVB4**r{lK^ntu;feHW(ZX05kzSZtiTCCYPata)#`5DE_Zc5mLI*aRB6r zS6A|{UmaP4tTe&^p+n;_YHFyNOi0|Ni0SofQV-+EB~GqiD`5Rm{H90wl{~~Hc@?L@ zD*Npx)+*K+UZ3E?-H0&+|ErlyD0F(3w+Mdw`~5$JTuowW@j2M~tDIPGq{!7=p$ozyFb9<4#|r&f(_gE$_jnplw<@p$OUJ3}~{g1)~ma6$j!b7hr% zvaaqq<>H1QcxA?`-Z+y~6S~M_isnSg5ZT|0;D6%IJc(4uz|8k+Ast@hx*}q{MYKk? z&~15)fXF0r9^g;RtcwrTuzt>mNuZR!V9zpeBQIPrH@$q~5{UJ}GLBOtk%Q=tFiD6C zf-7^-cZOhQhWvU4Q|k+Y!wg;-Yh+oNlL>PVVoX405WiJ>?Y>mP>p5;5doE=W3n#rB zMo76|t$?D((a3%VX%3=cfm{e@zJr~O5eS7(LNJv1Eye?=(=(xfMfh;>?<)vwS2wZd zl$+W5c%Z#;d4;iIWbiy#8Qv`%uR0)X6vuIjaQl%+BpThz`YJT(nNZL;2^#m&?rO)J%*#Jf zGmiqix=D%a`}(~8^x{T}SOqWyzv!8;Tkvn87P>Wn0PxDXLUvWWa*yBG0vyK$Y&Hb6 zl_TTP)u8*Y>~@dG;}>z3fwl0VGG^@z0d%ytG~ZtX?`Ku7uqsaeH#kLhoQyqbLxkkt z>^MkxqN#*oo6sOK1zaFJ_&X1b3WL^8;zGLaVMEPkLXBZ=cEScIcupQ`F6!R?3=pZ% zX~s+^ZtE$?-wVVX3>+Gj;xIfDB7v)^{y|MD}3;+C-pi#amJQgdN zSY+h2aSz=kaVE?wSg@>^i2KK+7E|K3@b_U}!8Mp2%!#IuD+95}bpHMs;rpu}j>k!?))9AdJ@N^Mie7C3P(>!NSo3DW*)e#v-h+8H{yYAVmo!SO#p0!ZtjFW?oyYv5r_#Ws zM-Z%Il~RdV_f1A)kj{h_+*obmaMjbZ4qzG|p6b^ov6Ofyv!aT4caTT0vwZ86w{|g6f<`OLrjKBMKMY{@B<-4aqH6Gp6j{)mK`ld9 z=YxiJ4NtckTBS1K^ecF?o8Z+h^et$ZSlhHpPs`cypMEHMjN{7;Slw>7{Y>emP$Xo9 z$PB)ti(GLwJQRF5TeYm&lFEdq!>Ke#GGQN>R<^avthb^EVPCDq8cMQ04q)b3(CHTNPN>Hg5S=pwiwex{SGvV%-X>5EGGiv?g?x&u7Scg{{mDluBkk({* zf(cs>YLvWiUwpUdR<9A1Gd2Znu6hjcrbj;hg0VS`7VqW^)xv+|ol8ht>lVkUr`l?@ z4{NIp#0>s|uwffxR1zCsx4t5^zO|kj(M$0bA5^7ksd8!2fDh0^v|bwF3=9`SJqFJ} z;^7Qz2OSBeqeI4b=0G?)FWmhlQMWO%*~#~{?(8)XieT0Ly4L#s>jiyJY3==HRb=~) zo?+$tfweh3DbG?*8Wtpgl|6<^8Hx_MrJ|LzPob~PJ&BOdB4~T~jvh-$<&AP&@^B>) zRw-b)sxSJ4eK6^y63GASyvVLmu(6YXZD2DH3{1JF?5)KU5W)vMk-z#w8y za;fJBa3v&KBeL86CovKV*}LqdtWxBWFxbKaRqK~(f?ka#ciRm+dyiY#f1|0$*; zp-Tt3y$W;n-d?PyJYRKoX`IwM19kil;VN^p9|^K58cKW5MiF4bec zG@zeMFDVk%H8p7wP7TKqSeRapmd0hHtV%ml--L2Z-F;VeUh(_`)zq19UXR{7DFT*Y zVSwcTKfNs8g*6GIBg!TS2jY`6WW!d5#7Jnp2v-UvS}pRQ5$$O{c%cXRO~1SfvwyS& zaO^IJkq{ZZ{R!rgrAddwF*)Mvg>h^~>9T;Eaw+cIkr)XVWGG_gRsuz@^Miipxy=n^ zA==q{taUGn&Q9gmwV>Y?PIZqzbbD4hMHN_nd(mzr2XvxoFm)6{E(^S`rbtvr8zIBl z!sedyrXsrv$Ul5h)9E4nx}so%h(+PFHOv{YqakKlbRQh<@9Z49E#!{}omkq8E`wi^ z%L4DqD3O@DKapo+az|-L=qZqgdaqY`^w;TwIzieVdbQv7Im?BxYl52F7;UrZB+{

*8-{^D9d9Nc|7|XjeYy*)JX-&&MgP_ zZBk|~oSfd;om=1ZPAso1YQAZf6%=kp57xBsm$UR)H&POB+=#ysFIYlC>b2C=)H`?5 z($drK-MxGF)~)2^%YtQOWZaI6i@Sc^`!O#0UTRvzk@8Rwet=@ep# zk|Uwh1pR%b_^SK<-veF^Q3II0pvhya~I6q^iO**t8LrA95coh6$F3 z(%y$jhK&vx_~+i<9*38$YIQxQlMvE~S* z;YzbjYqR7!&n$hvhOd8+U&2uL_#e^5EQD!OD0*Ce~C2^&CtiwJ#usZ4OFI{J}1OAd% z<@AA71{bUJimOp6%ck0~!;Gs%o?oX0mJKP4_IY4+Vy2aF)eRG?^vcwsFs^c+u#&L6 zU;*@YikR8-C|^EuS8DYeHo5z-4o{jD#+9lDc@+#j7U+2_;a$jNh2$+@#t+tK9Se`O z0JJ|)8dnc>IAE7yX?7@8bdJ_JuxOr1KLP7KswpS$R_Ab=a(ZM$fzW{bMTTH`C{=Wh zn(R{;6bw^=H-4~IVd93Vaz)++%IT3gv09#~*s$0Ms%zWKES6>^0NsADCI|=Alq{5QGz9*vG&y~8~+uS!sM{;mM=XAlOQl=Tu=#_Eo zOu*Wpd@N-N4Ayif;qwt#>QP{{?D%M9ly5egv%NZ{N1TLy;=C$Fk4h0*?(hZ`0RD zKa0U?_4Ut;L$?ssJ3vW+>^}G>9 zfqd9Kc1vVnO|Q=};VN^Nr(20m_vYBatQ#h))#-{vz~Y8l1HRa}vGDZbQG*bHTcM4~ zBN6h8Z)+OM0jg(bX9wO!B*`y=Az0q~P1H90-8-#f z1=#1&^75BlSs5Cw(zMay)*xT4ekci@HU2!Sl6XoW_PFXW1>{&+vx0--(Aby*vQEb5 z&jz52psU6W^BhC4c7bcOhF9Z{oBTprt9n(M@-SSDpe{>@u0(k#I04Iu*sJlBa)F_6 zcHqc!X~8vCfAWhEYmtcVBbyAtBIkQ3Jq@o1ONY+bxFq3W0Vp01^;Kueg;eXl%225# zO<+dEV2!5-+@D#&YycwW+7LmObpR|ezU-Sq$7S6;HxeFaN8x%bm(`h_ZNm6@u#Oi4 zRpg9Ttb*07!@Ce*;RLKtkvt670Hcj1ST-XPsF0NqTy{UBXoU+_4yqT#Fyj zV|jf{g7sLh=R$z>=R3iUDdSKiS<2c5<<8My=bKU}R1Gh|7)B&i|DU|G`)Ok9!Z^o| zSFPg5@f?pDDi;sQn@lnVJUXG$BIO{p`}7&ssYy`~B4C z1er~#?~xr5fHA?kp1Oa|8@+zNhoZ?^#m6fe$Ko^#vQA}KW^#jo>iN1lvMuttw5I`_ zPx_0#1^cF#fK}aMFtn;3i1ltw3M{e1qPd&y!QvIoW0=8U^vN8jw|!uxLaHQ=IZ+=m z%MMZ#pBb)s=iu>6f_0;j;#61=u-**|s$Iz@&Mh0PP$VKLJIs5?x=AH51pcnkmO8Wn zy%JzKu|#7mUB&QppKDlSvPM;dbp#<RYJLXR*>W3X2mgJIIu2DgB9KI(16wPKK6Bq(c52n)@79zs1nKq zD+n&MpqMMK!+7sCNrRQ_vi?WG*7j=%SF{#M3v(uv1=jNX7{}E^mMoTe9VwPVgWc;i z{4NVD;;3CJr;)NWz1F!hr}|n)X(bfF;&K_`=M&Z6nK4QxFJlEV?KHZN7UJ!f25Yw0 z?~ii0p>za)3&St9%~*oksp#WpO>Tu0#5vfWXZ8L$iMuBFUmGHdi($l$`cC5{tr~bX z{1x)T??mO2vVD>=1iC&Y&u9@$}YLvf~3!KC~x20hDusBkc$<$<=FtZ zuLs3Ln#x!zH_!YmefN@0dDygP>$Jv_UchCW@-8cxw0NWJ&}u@f0JuG z5q%VQM52lWW4w`Se7f@-CWs)r-DNzMv%|<;++@xBJtL~WtD?L{5 zN|=m%&!Q~TbXj5|w4==^Yqah>fTvdhSrxsUf@=5;E-kNh7W;7ah}udxKnYd|!R7r< ztYoXz@1*){9Jh?qzW-6P*F!2LmnxV%{0a!#SsW$>96ep|GvW6ki!5(n&cpe9EGaq+ zQL$>Itq%7Tf>ohljn-7`Jus}1_CCtv=fkSiQQAsKJfOjBIu_S&Lwsn!M+KIVhV5&W zf+ZG11#Z&qc5RZ@i(!~9loa}&Fx`>h*bmk+2Wvf4v~3MPGFXa=p_sL1v`AWuE|gwW zzsJwS(^f*W9a*~fATSsgeAHkWNqlcm4Az>$os$1r(ZhHU?4uP&8Q9&=VS$z_FS3C} z7B(!nzpGs`-80^Z&mDFXR*eOgQz2Lii(xWp%`r?`??!=NSxl=QrRuSFb~j%g zUkC)^n*oO)_v48%epnrxq0H}pDhG=cLxrodtPH1DU-rRtu5j~&x%{M_6id~0{lVR< z!!ai~G9|&ucad&_=O2?{*K%2^Mxakp`n3+p+uily_|kJlp{HK#UCAtL?UQ6;DYX|YOGO! zwMOyUKu0XGf%EP>~g- z-&QJuHLbjne=!F}yBgFQ8?rHC;g-`M{32V7EYgz2B8A_$JudJa+1d4mwRJKYtYu8eDZg`8sst>^*<4`faX+~cy0hZ?dtBY zTdU|(sEP9U57XD=4x8X+2UZ+)UEP;d5IirM@PzDjl3OE;cv)Q50z*X;t}S79n=x$7zLbqd$Q)9d^i1c zrH3BN20v4q*=NW;{-k>z<=djxa zLHykg7Ox(L2LcY02v@sqB(17CrYXybYmG71WPO1^Qr62k9++7vPd&bQ^pJK~l^llk zVOd%{>_VuhheP60jtkJwax7ZsLiP?TY5~pja*}&W=dQdQ7K_1FQoiVTeHE@t8m23v zVs)i+RrVAnpSjo(LA!2*hXxDB^jx?Bj(II4>)>60H8!}u*$~4M~d8JfzwSTyaNwg9cx1qu+oy(Yp7p& zC1X6tl-au{g$JRyQmGr&x~YWs7z?Cp7K#Uam0U{qp!8IU_HiD4d$|%*mcwf<=E=}& z;&mgEb~0|UCYbA^8R=?Es~hM2%Zs}Af%P{+n~qR1Q{}Rv#rURhkDhf+!z-M< z6VN03z*;BbnJHdGuom!O}n;ohpnbn(24{I|o=@D6agziTZnb zI1_HL)!k=3_{N&6?W-&|Udij-OYu(*upS{^RItne)HmzkOvo6liV4=H8KQ}v{Hd^9 zm7In>2UkRX_Z6G0ga2Y-E$f@YieBY=NAoW;x`wmoN(voEFqjLhPgeQ|1(;RCgYTZP zQLd560~X6GosO8Mw)Q-_d#>J{%o$o2;zdnb13=B21MkLSJ4#UzRkILl-xbcPwzl>> ze)sj9V7=`EAcWmRL*AlgthOIbA0=S1(;Jp}ZEfv&$hDdFD!e2+uBGzJ8!*SQ}^;U*(%uZPceGnxSHi=Lf5-TfoTYZXl3R4t{yZIF(2 zN7|M2|GB%mmp0QZo_>v8+ip9%EoFj2;Ox~4iKc0yX}+2SH4_6dE1HDN7-J}6HkcrY zibx4UYbc>tUSv{ldXWq1&tcm_`t!uctQfz3E zOKW`OH^VztrCwY?73W%WdKSA$6+E%y6TQYN&r>`h#3#yKY-oNVIxA-~XZugsc`gdQ z{^}s}N=6J9sj1*E6QqKD4x5y2UwmNYDe6}OMTQRgIc##N(?6BTOxe=F-nU+=ElBnw zsu2NMQ&ZKPsN|_=I|RnE;#XPx+d9}M`na2P)q&W+pe!3`0hY14;AE@#KWbrp>%?SA zXEN&`?pl#$CF`VNynr+-c2@^@GnTwQQNCgWqkfO}(vQLTtB-6Se%G)B3ADYtA|fke zhM0mm>(D__$y6$pOeV8QkxHzAQ~sLh!`ofP)$Iq`FMV8M?`1Up>Jt0bi?#S_G#XvP z5sklGGxNV(&QiylA)#b3D^@;yjbl2G2xn|;8pja|^us>^SP79yOI3Z}ipE?KuCbEXPTxiPT@5j%)=<2U3WyBp$VgsLu zLZ)AY%TH5&Cy;S~+!G4T&+Dkv7+7&JUs2yS=Yi^2=t{!Cv-jW0r7OG_i=)`U2Uu99 z7dl1eUj|a9q9gPx@aP};@>LzG%3CuOY^ORZ;rEl=9q)>wD=y9k-tp2&e|`OeaWz87 zPNN8HCzDB3QRP!8rt1^84t%+ByAzKeaBk+H$DnfPN@B60F2~2sU>2l1pfs4U5W!cL zh+fB-*x7Dqo`)kH=vj#91`>&l5ulRo zarN}=uiScs|9R4lp~0?+4UL=?71oblhY=!{M*7C6qkJmMD5P}MVeR9S2D;l*fn0L? zsC61KNkfy1cNDK=P!?TDFE&(k(uKH5Sn<9KqeKvvkJ6=DTYe2YzI+B4tNe}tF6>$O zE2~4leo8W3$s#tC^B@Qtr_PYF%}5<;^(bc;!w8m^>uHq?Ms0C1-HmhLOGRuLlRGG?fC?~D!(8Q3%6&nRXYRBz5ibS%$A z46IL`)79PUJ&FU?V_1-(#CjPu^B3MvVBi-mDT_$URaTP*x*q@+Q;~p(9xAG-+t@mW zk=zFDZynN=GK>r@Yq4&bWf2?t;=p@FZUJT#1DZ)sXt=GI7HO`VvS4reLtgc!r%Gaj zY zvSmU%+OwNB^R$x6ew^aa;&o3Bs(r=~Q{k8l0nBZ*<HJN00#8?FZy&Rb*pod|+Y)YsB%SGE2 zs9Z_Y9$3a!9V^*Z+_lW>Fm3HOHleS28RF^+?%!4{uN27?D?VkHK8245A{~>BhI)3OiL`9k;{zd<$dk zPj}jm^2PLsspwfmx{r8D+GDULyUSE!^}UO)n)v~11?0ZoI}BAAU2K%|UJMpBbzlv* zktAS@F5dImY&N-%VHE&F$87_*BLV@szIyupDceRr<d1VExJ6^ z7+nZu!aA@Hp2JY_?4kiznz{Ck0G}rm@?aYDMJ7BR&hoC?aLL>k8FVF+v!Rv>U%%#h zpN_UQXouf;rG%9+6bcM!F1{X=N@vH%Im}PRV(ks6Ib)&<#ktfBilunAl?{Z@m5luy zzK0Lj!D>uM{jcalJXJ5vHKn3%M{{MIK}7QR!zhc;Knh*SH%1`jcruxbnJj}xvVf= zEO7zWMR5pR5(CS@bi5NpSF(*!Fa!q^M_$??P-9pnT6i*8O!3#?~^(v`fSU0fyLfn18`<8B`nDVqZky00PA2#(wQ^_tWVDX9^D`hSJ!&Y zA7uts-)l20H#c>rR$-tvBkgD4Zc54pGeJSsJ9a2bDLGi+sUl8GD0z&6!C5x2Wm;w> z08HtXl_``IcTI(@%xwK5dg+>S_PUFmujDWb_0IeOwpk{y8Yk#C)s~Q!=B?*m zDOZlp2zwMtmkh%O=>4gOSX}jRI07xYn#D9ZVE=KDp*Bu>v=B>3yRRV#=YoNyIR|Pq zthUUAg7d=>htJ6WJHhQLmy??zAepYP?EZ6Kw}55qg+^5&b7F#l7&Gg+3{ub3i@{hL zO;HE*6i>zB72}C4Ul9FIBIujTYv}UPdBO3M+zsf)ur*DtW?%pkBaB5qn;#?wmIFcI zKsd4v&c|WAxU++YFvbD+6@`ug=)H9htl^-dE|&0$Te8dU4H1RqT@)vE!C^r98L#Hz z-3l8TQKIZVJv%-)pnw-+O#nQ)rn+$leGL?Kv4jM-d&|~H9EC<;MdjZ75fddBHnydq z-H0@MgrFC@LwkDekmNq@#O?m>RXqBqtSafdwY9Z46IgM%fn|k~;5;~l;yD6T4#gtq z?R?Y+BRr@|q1RQ!#}8GvdwgNx)e>mCpgkbkvQ-vz&!WFTT>2@{rY^7ULU0mfo)YKx z2}IyW)&EW0C|@Z2_+_vBC9!v*Z%F4uTc&^gWVRc?n)x4j=NHmOzQytM-?p{4+wI+M zcUxL0`I47m6U|Cuf*F%o68r~iDWw*rQyYRz7J?+mX1GZ1MMxgRhx@$oqTp>|5rhVW z8&HZ57DW1>_C-+?A6xpOFZy(UGZQucO!CXj$TFVp3I&(ln$Mnde&>A8Im%QsM>^8! z>BskNZnxWJ>u9d2C;illOS<9eYW6~*_}N-FHUU%vx(@(Y2p zT{KD@MboVEd|hc?*y{Sr-Rx52v2J1^o=l}VY?&n6THCOKi;H$fo#~)>*#1{ZGvf>j zyc5E5nE`7a&nagi`YOm13jDaR&WN+KVJR* z@!23aUi@t;{eVNq{VtoFSi{`?*N%iu;qk@UMTNrYW-I=#0TdqV;U}&P_gp~ZKnwXg zc?{OpfM_lxcSy3u_q9j3cG0wvK+M?Xe&Ha?o(yAiF#F9tvV786htcO6Zr&KNFkcmc z)hV`cI+cM%xccF$p13kx$G|{_z?7NHnDR~Bm$4%zUkot|`MXb(9S-8@aaPtIpSZ}B z^iFi&f9Y-mCpe+qfE9VjDHY9hq7fFulWX`* z3$P+t;mos#zp}ZpH^~B_t`JbiiHCm8A)h8nzph^5Z4|xfW zW^=#n?(sWD9e%&V=Nt8fLS}Qw?;vqdUtAe}#Kvi0nKI~;qLQ3^ZE2!`2%=8n>L2X0 zheP>EU7TcBX0$tF4;}=F&(dbAtvId9C~QE7s~8{Z@}Srl&(WeAR%iZ7oPgds6|70g zM@8iCm~52mU211og#iwdRDN3YSCp8_^KZzv|% z`|~Nb=fNGlab>uRfz!d-%6^iQ*;Jm@qzS#eoa7>`9Wk+xl|L*L%xj0w?2zwGrBXr! z`GQiL&zub{|D<(gR(w?s*7z_OhfRjs=4$En_u^&YD|W<#!O~pqGq_{Won0vt2~AtQ zEiH{M7bAj|>1{-xhQ+KnynlT3f=>z+wG}gR^|ZOeCVzTQ30Ca^K9NX7P)|vgOAUeF zqxmtCK`}jDSiaO7so%3H34}Y1M*1NKR;X-#X6Sw7hl9dd8{Y+kbIb~hn zsa)ODAyR@QC3K2mng zzp=p5|Gq3(dgbauoh4H}SY~Lfrb+UAg6AtPfaB#m%psVa34!OpfzR)SKR`b*hcJ@l6PJz zLhQ`shRJ>^u=-Wdj+RF&3`U$03>AB<19@;lhjyus$xwkM+cN!wFCYYW;(}@(OG>gr zC@~J1&A+N1wz6VYTd*E|VGwF1$=uLkF;|!M$rXu&y%b!TCMPE|nT(5eF-=llzQ6!p zbfKq{I1$1n0%OD6+4=R6V~`>h?io(+rDPpiYQL_2|4 zS{1OKPfw5B@9zg&ko2X$0#8yoT7vVqm>S9VfdYJPigH6Un9fCQGpsRB6Fi5-55gtf)Gb$r%XE>jwzpy0 zgSW;bn~RG3u6U)ZZIX7oU4*L<1^0iFbYo+WeZbgPZT;2uJMt&qE97bjH!tdzE5i?E zq*$tf6+qOOV-w*g)(l6v_OI(DNwVH(91X`w3b1roz}59{^~=@On>i}? z>dit`CS71GyT&|Q2g9iZ77DyE5r42QmoUgmr|cx4Azt;3TGL#7)EI| zq>eKI>&=jD7gK_O0*3t=Am-Nk5+3}n*se4Q*SjJ&( zE+fW|il($_R@)f2pV2 zPV++B?ENza_YS4{=$>6)>tssXm0BX9DPtX_1It?s)=Fok?cd8@P=?C|)H@v68rDd+ zkLuoOE1I^?;z4*WxVm})R#Q!(kS=*oqZlmWvbeHzCdFxVx@%-9`0|cf(5zU0>_C%- z4a3)2u=YIq8+R^YcWa7-rUA+id;y(vMPTg$?CEJ%kO!ixR3{A*fmj|`l&c!wyRfZg zinVu+xS|`+$puTG^d#XTu8bd|D8o$vm@;m{PEjutcRY+LSm5ASdCPllGY9f$J>lxs z<#WIl-FQxJ;hhewWWKiHh^%3-fz(V=h{xj#C3H5M#tE8`8;qdat>h-NTf2ARqRf>( z$LLEK-FQyzU?s3x0xaX)uzbfzRA((b2qx0`TS7LCpJ3;!P2Yop(oMKF?ZA>#cb$$q zm#`bPMM44=-D|D5qm%VnQ|rU9Si;dxc!U5vf#r0Bjm5KpE&O5S31N;)SDD9|L#;}| znjin730VITGpv2+b1b#kD93YZ153PR!WGv?Pu!OAgQO{Cz>vS}apPXq?+z zt*VeJy*&Fmnp>YXC<%p~!`d_qXMscgj-?jc?V6-m#9FEyDqLPpvr?#gYph`FuuSZ( zTM(Yg*>v|w($`c6R(5e#o1)`i!1%ldHHR?_iDRkfg)76|8dI$5!Ag2KbViBIRRYU6 zAyPCqwK6v)+3M=*`uY?FSP%F>jEg8*B-!x7FTwd)Wv=F%uy=sE+vkO=np3Rmz>3F{ zf_R^P(@19@R)@pk==$7ovpQLQGAF>SD9?J|5^`Bo??#UUgSs7eE|p?^huTssQ)4w_ z&~glXf?Zm$hGDKVBRaJ?@s@b8P71IDx`D#d_FQ$a<~`8wxU-s(&{S~@dO;8bJEGBW zcyl(WVh^_#d0@dV}zF_{H^Y$DOt6DAO>I zP2)@*mmUo>C^1g1vTjgc7jqjjSW%SW-pt#F4;#9l30MIc`gy_Xzw99Yt>l#v-lWaQ z)owT$We02SY@HGOi9X<-V%hthd^N`#K+2@PpaTD?`gDwCnWNCHHs!HFEpBQ-_nG;e(C` zoGTO~@a7;KphN}LpMnsGlkdZkx}v)?F9;#%Mg)ttS}%FH+H=HoT(Lhue{HiI6%9Or z>hC4>sn5*?)+ERYl_F8}(#x%?5+MK_=m2J8*|YvtRI(QDMs z!{Qt#Sag=ry$x6JZ51kSp%}|@HdvD|Re~r){=uza3BrO5&ig-)%xHWRE(j~6&z-k3 zRqsf7VFfy`x1kpEK!^9T8RE{G575i}V4ZjeqIv zj!A>llJ#wzGu~xAF-^>?rjDT8gybqzQkY!Xav+Cwf~9^Xh5>GqLu=-*^qo+cGPXfS z*F*VtO zyiu}9sbFEsqnnEi!I+`+shQ$9O_1G7dFV+SdSzvh*Nl-xhkM4G1Xj;Lm!a-`6r|ne z<*EficBlS^z7#%K)H4(I3&tuDS5G+?bUwo-tD_UX7~#rXv575p3`J~s(2N7CG=nB6 zFx4!8<4gy-MymTNG+%6DOZ|i)Hbg}!+Z3>D5m4B`@&&^H(m7nP^iNYZp}oCEY*@wk zBwWP1{$jCQE|=XDvt|XxqVfZxC5zTGNqnuBE7RGKw>TTvCfe2X6?g;*-K zPMvUh6mO5bx~qfQI;$#4J^ zD>m_ai4BA+58fx?dP#;_3o7C)NNxZ>4_H2O_zhm}$5R|Sb+qqa+~4{(ukDL*rE2rI zvdx#kqOJuYNH4`a4_JY<71C&lNFsdypq{vn#nleFNY3&9#g#1!a)l!%pU4LUzVau1 z;6O5wg7fK1i>u#7Y{2G0Gdm7C|4~OQ#fJ+^)=vLg-*#0nJM0fhTYKJ(OH}0%eNBNx%b08sypOpm+uGUH`H9<;NA6frEFggs@W_6y3 zaKz$jv-a^PRiktf3EqLFfVhTq>kSQmSo$yw5ca9|h&=h}R~OAaddBC3KPT2W9G-xA z4Y}1sxZe0ZE?Y?^J)(v*NdIqbGtWRc+ zJ8Of*tzx~yOy?9p%%gq(E!oSJQ3njZN>4w3{$YsxZiw8EC&{lTJ2<4QPgdNy5m?|@ zp5qg-Fm?Bo$9Oumt^sRe6D<2gI`eU2ymWr0r(b34ON1{%7yFo_TNNkPQ&Y#C>j>}r zNCR@0StE@)K;pazy76hE!}dm%dG%tNoLN`cm8?;VtA>xz$6EIToLy%6 zI9xpoN{(M=ubc!wFIW1K*((Si?AulP1w3Xq{DC}ban-PP{&)yL95gOyEsNmc?ynS( zt5yuS0WiP{!^dsGvX2ijPgU;d+8E7E`m19WL-p#rqlXP2ThT!fClQN&@wbhkQ#?I|3#&Z~xxh6nbLJIV1*cPGEv$G_F9#L;Y(!kPjn$~WP(9Vuo@JdaBM3gFv}T7c279jWz~YeLFM!i!@-an5>fleh z1PBMXag6TNhMcTiMdxuvY8bkEF0fyPu;-wKi?L&i>ZT34t#{C8Y>e%E zB)2}F)!sc3qN0=6<5?_E1R*D8e2wkI%8UW%=gR(qqs8j)wc^e!y6Hx@6|J+hK4P?! zQk_u|EwJKrg#=yewE3N=uwWI8$7RTjXC~4UdbzTv;pDGvZI1>RxVqY9#hqD?ts~#l z0YJ3^#GJ$MEUXB~vtbE5ixEw*{E3BR1z0SO_dI2@J!3akV{mcc=Bd9JkM8tzS#cbm z=bn-PvAXr$*_&ec_7mbz^I)CL7B|AFc~H2(ii2}0ogOy`))Jg*2ZmviXUo^$QcV{qh5rxlyPcuL$; zbtSGcC|DcW=HQkDBK-oCOLH3hPLPzcWNJ9oCCd_|S-{G?XrG?`U~rie;mV3l;Arx+ z-70@2vIL)F+F(^?=@&#OeG)K$<)RJNM>}hg9{ys#p}yUUO`wTe2lkeq5+4}Pr{*T9 zs6ZQ`;@7xyjxKx-Rwc70*}z(Zp^5Q{ab~c_#{hbJA9w5Xlb&mby%txfn*8^wCVvTx zwpdwZqmv4Ow@^B>a^#QPkwVoVtQC+wZntOAu+?uxzj9TlCaAIsryk$x(c$oeR!e6< ze@fg>A-D9F819seXn&*-BnYRIVQK8(YaJv}zPG*n-E=(u5V)$IDA6{Ldx9L2gQ`E`RO z8;lk%uxX=p#rTuw>T_kq=$n3O{ja_2d1)gJ!>d)dYW*3l?H+{kKo0{G+buLJd)Po} zA?D<%hn-Wd13k1?>mlo@)nOf`!EQTjpcB>Xw@*IK$PXOoU`81x#m7xY1un` z_7*wJQ>i4>!Kw2@c_vfy$q5KG3Jt9H>pX-iy(F@$w9 zP3Tah{lt0_1y^uRtO|a;%&UK1V}^LuVN97wiadhvumi+m8*?fyybJj?vWwn5M)C38 zhx7f5gVUoMm+_W5!K_V)j?XNK$BSppvM6nvxC4wCT^e|UCr8)zp^Kd$RyGLaoieM8 z7mYUfKb~V^fxhz{NLtgLk-5ic zfAGvtT%*Wea#@AjbTMXJ(XyC9OV=EJe-4_P=mJbHa=?d!7F$#O5v8aoCEfD5+YV|K ztq%qG(;QS@&EF&asw?z|rCDa;`}c}fP+fuvBZuO(9#3Flp zR~224nb-8ZnOo<~)>uhv#*Y@+*aM9s9d{(<52 zsL3XObm9f4xXr6D^EwNar)BL-^sfq%v!@~Ww>*~ULv~{!uW?J$x43#ef4uJl*M`#l z43H%zLh#zmYfj_#H%jm*2u`nqtOX1^rLFtI`9FTXKg94_E=D|ysGHaWJ1}Q z$pH`P19o}IHSA%@%rxNkXBLd4-izLoqm#Bt z@I?qYZ6Lz0Yh`&yyrT1}SKyfhIJB#2coqd)pOZ`1h@}7p2c9e%AM7OzFLE3<+jAAN zt-Op$u^O?$us>uW)A(So-HGFx`kr;o_E(BuC6@XF5w=6Xc0amIyy`hT^2{*}B3X2a zS`lM`bVesuN&v#_@3hUaoicbDAMC|aYiVkN1!ud`EGz+=RgRu1A;?lhOOv-1=iXee zLlpHj(%H?VABpl)iET}Z`oP;TXT=hs1KB~bEtyCUC z_9lcN=I!u`woUXr?rDX5C>yqYUZrYm*wd$?vsoX?@RhOJYHVD*dAPm8cS{(v%Q`1`~ZL;|FJ{16^66?SuV|hx0SzGwe#dxss1W zA=Kuy^hQjyaUPmWSkgvwJiP}8p_KQiS0aKp2@r8339BG7YDxb3JXts!vRJtncuZ2= zJ|N;2GcsSRn?{oLEhi!ZtgBNWfIWmo?`VF7>h{4owSHm&BiM_#bU++77-9;{?%Y_r zD-bQsL9_QDomYbdO*i!(DlKQ}4XGi<7A!RmlquGBHmUDH+Lk(S2iF7aWz|RGm1E0j z1?EyLh&yV38JPMWoToARfx`&)v@MsH9*JSY!k1>7cey5J6|H#P%RI$145a#?l4@e% zig7Cpzp;n@(+}2jK5R_oPFb*Dfl?p`Q`TCT8`4)5)QwwVV+C{q8{dAY|BKrp7EA6U zUJW>E#w|tHu>wZI!p5@7tYjmx%I0l-G)7OjKQd-a_rzZ7~*N! zw5^Zic0yHvm|?~(rDk4;sipV(Wrmk9#=ul!$!6_W#4Ca>Kva!eNp@o8MDSbRHDYZ* zP_M&id{-^;&8>(g`xW65s>Us(a=_o8m#f&qPF~wYkG(ygnT8J8uLv#RSsdrUFKG8DQmQvLSPR=b&-BP)A z+0GynuL#aCG{>x4LOI`TXCR${-~zXp1R)yS$_tnvVi8;^x7afgI|Zi4=T3{2iLU1!serUGynhq07*qoM6N<$f|=!8MgRZ+ literal 0 HcmV?d00001 diff --git a/scripts/v.dissolve/v_dissolve_zipcodes.png b/scripts/v.dissolve/v_dissolve_zipcodes.png new file mode 100644 index 0000000000000000000000000000000000000000..50a28233d20fa8764ce342b80043fb2bef49be2f GIT binary patch literal 30257 zcmV)RK(oJzP)#B_|~)BqSsyBqS*( zCn6#uA|fIrBP1juBve-ZjUC71aXH709sgBTssO4YA|g5CaaI3S9mkCTs;W40;>H|~ z9FE40#vE}u<0B#^;y7|6BO%GJ~JduPvAg7r$IxlB_ty?Eg~W* zD?LA(LqxMJBPlo`LrF@`KR}{LNy$qwC_W)hMMk_#P1`CYBu`Q7B`_v1Au+v&9mIDMc+xOVWEsEpQ&2PaZ~$ z9mZZCej`sipK~6pS~8MQFGC?YP9SG}9j_){Kj3gUB`r80E-_>uh?Hy|+HE+4VIOc+ zAc#yZZ4YfPlTch0U5^Hjz+yCh3Vw$jy;?ddV^1F(Kt3%yVpbn=L`Jy>jEq}IArf9* zX)!i;9;WS4QHy3DRQpshVlXZ%HlhNeA9`DGLo6U@P9B+WFfA%%J}L@-e;zhCHgGeV z1DZRBI7>{{$!0bqH7r&xFmxYSVk1irZf+M$OdY_8ksfOwie-2+I#1_M+)Yh~BVD#$ zGlLzsOxR2`x?nVYJRV$nr~s%mF-k5}Ef-vP9V4( zg&c{$e|`&+TqpvcpCZPDsxew1M_Ycb9b|46Z`?NvnX1h-X~P_hQ!tnvZT+nKHW<1y(&`Vzaa&Zo7Ym%xkE-u zk%lzy$Gu}4{2PNk;~8^4(n?9Oisbp|oO91T_uhN=29goSq|1$3e!KdkcV>8eX!IUI zbj}!e$_pq80)>-&{%5gx!C6>XxXA0AoD|-d^%@?bpNr@)PI+}_o^q#(3oW_$nJ>Hu z0D<8#`qhZ*obo^fs`u@ks-jp>NG+l@qti~ep8Gm?s-o730>w#c5lwk;eeP?cRshPx z5dDZmBjYIC#;IzxoTL`fz%b4ix}`XGkrgZc$k1QmhjsWlkosM`{sOi^7vzhpJK^KEMN(&T%3P z%sA*bCG>*onD0U>vE!CPQweIes@cg>9p9oK1PV$$bDW+*DSw23M+Y{G>TGKSbd^d4kvnagy z1JOh_DCD{-(&KTt6XF{G|Gbkl!AxzeM$SutDi&o($Tub)3-Mx%rfNv6CL7FJ$&pkx zCEtc4t|Vxx=B{!7Ie3xSxqC8MMrx5*wUY~3xfi68j3$`x z6qD`6dV!cM_f9Q3cX4NGtjV^0tP6!@6nb3duc0SBrLfNUWZf zSXX6yItWbBEa2cX-FC0=vWwL_^qrm=nqa=$V)Z;?>1v5tK$n9O;y^RsR;+ye6kl)9 zhvPkyv_E}k#VR5!8$u=#6l&4_^uTh>JeGoE*&vQWz#OS{H^nk6Vu?~q7NZYybi>wQ z;%GaXY*39=Gy0&=tq+69Y+JDkNXy0|MjtRkJ=1}s{b7nPNXbT%%Z6!$dGHv*hyIjU z^#?D!RBR0tWB9XwC06YSGbUy!BMy*QzxA`ksw=jp<|!i%2qSpkS}YYg**r1{%7}x* zC<1LttcYwXry9?Q)P5KYl-3>6@{MYya+GA^-UrHvwr5glbshJ`7i3HGqnX&Ld1~qH z2hSvBMB8_=+<4k#?mb@!h{N^blxmi-xVafIC^90=Y}uRpqfVf3QFjr@Qy?(ooKcXB zYs~Cgt9;Cr)`)h;;l=x_pjABolZUFAc`^y$YX5QH)! zA#AGo3&X~hewihw#KQz$m)lJvfwpaxI?E}r$%fr{f0AKoKwe}-@kl3` zWmjffw!9zoQ7m3;rL>d=&}7X3E*s4nb7;Mh7@;xd0c6^U^MyA=&xkIXtQjbV`&PWu zWX(V?8J)d9YTwOd%`CY!AWT?9ta8^BQ?!pbU}9w~pjIlx=tY$Q!BxK4PqRYcikDum z^Gs7?Yf!|}Sk59D@l@QiH8(s)VQUb?O0%GB9mJak@Jvu0Z+o%QtPi8gZHq0v{V+~z zMEf#>BRS1iW0n$2E~ z%sGeY%3QlK+RNup47&{pa8n&`vvk;xLtr;O>6#1%d96kLcsns{fnmQ+D)|uYP}_?s z9JMQQER!bnH&y5*DBCaswcX7iT3EAIvE`w@mOU76f1ZLTps01)r*YAbch6JQvH{Na zWA9PE9#a#i7E_B7qV@5#znMFP{cRw4WTDm;)mnC74B@tSWJU0|bg&7Q%ven=sKnJG85CUR48?~a#)uH-=A2jh$pq| zSYjpccz0)`Bo>!!OE=56tMVeA)FN(QVrB72k{*@B(pXt6NqWYMcos~gNG;-4C6*k; zPDQL7Pl<1&3J2RN|9dc-vY1+QU5zbDEIA~CR7oriZ0)Udh5z*icZ+EhK+GfLUsQE2gV8>)_D6rWWCWej&Gb#krIc6FDu)Ba%;& z)_`!5M}sY?krk-c8VwX`al)9RnJsHDjij7QGN`NAI~4&+6`yEB5~;AnGpV^*N3yll z3WY*T_y`w$Q%m52#xoyVc`j{=Ekm&}xVF9N=kB~_yT325uB}Q4ec~D7jVvb=K zD^e%;wh>NQPprj5OgZSg_I*t_8BuqAl}^ITPn>1Z9PP-gRm+)`#J9yCAAuRVy{+F& zH7xob;#Fs2N&YoU++SKF*5TQ=Z-;waEqWKRuZ%$T)#=Ex}?DD=S_jP;mHd zPi*wi=j|U3G%Wg_;P(#2`rZ6D>8>FijYgAEd|v$Z?X$Sxn?7*AN~~e**6Xs_ym8gY z^9^er%+Ddb$T{Ot%SBf{^a(e<*ZZQ1xxP-|i%3fz%s)jyh{dB8snsvW%0qnB6*<%n zGvOnv-Xx;p*J71gw0mei^ZMRR4bQOIp;$Y8e}$(7VVwO~EUWeP4`70}56xzxv9<6C zPdXH9%c?g%hzmq2X6b6VsTXf@?e1+X)DN)Qp;&J%w9h)|f(K+_K8?jaq%BEwf_$|Ktk$BexESWP$tY057 zN#ENvW}7VlZy?sg(u66Cgp%3~lVTKD-(O)+v&Tk=#qjy8#;h|%p-!o3I2+GN+(@i< zZ+4b99j zbf=ju!~>Q-Rq1PAusyue7RGelM{X<9V?hmg2$MF|fDmFzLO-lnOe` zYIJk-yJVnPe0_NE#$SCxL!eax)~2uiypL*5nU$sk_=c_tTQo;|C|1L^B6|Y0)R;ew zi2DL-Su*Kd#%YvpqcKt}7q%r+#d^KDwO%Q)w1$8s#53vrv0RvLqcP5_X{^69SFGoL z-+Cnz2AiDW-!kn{YK?~e4C4V~|7Y)7e$raEcv>H~tyNpK$0lS={f^nNc?g6fuc+h* zsNFtF0^Fo=P(ol*NpAQ9nCL)E&b&D$=8Q;BoZWQlz>cn_POZ!x7~`#zVmr9sw*lS2 z-t2EbBsXQRL2E0827a#B@3+?SjdvDUJSSM~gQjczbK|!e=+vI?4Wj{nPxg;~F}ckB z0?X&EHBCwXdtT%LW$4tpT%!ShU-lFn<_-hv0M$1h6{6yOj`_|_zoro~W?1jRegSr7 zywe_Q4pme4{ip=@z0|QJi!|VejRw5#_V5LS$2$(J8nSR0a{DD_$xz2unx00X*8qav zRL-yr7kE#=nqeXF@({;CsdCiQ!ChNL1OtZZuR~OhjwH007;0-S71*{<7Y)Ep_z``$k+CgBw%tehAG3mH`iB{GRFZ!wa{%p3$P9_iILdfH#9J2 zsA-f_IrLe$O-TC$rAG0U0gKD?t|?Q%A33kuS(!~)G*5kHa>aBqzrjU0E(X^V0^=x) z%zc3KZ|8q+@2yrH-C&>Lz35Rj);p{uRcbh~n!?d*K`XH4?qbvFMz!r~!|9OQt8q3b zE$CL%JXX!FD0PE*?5?VtyD!g=w%-ZhHEO`CojvFu93gL&Je4D?;e-S5OI|=z8hOJ7 zFTWD9P$=x@atVT43kwX=+`LKy!<8#n*VwMEz6*i%gQcS$b1Wd?9^6afy&`;Ugy$nY z&qj^ZUL045{|R*2ca>=J8ZS3ltO&eQR{Q*%Z$>S$TmVerC0p|?HvFVPVA93>Mp7fN z$^=9cE%xJzrKvY_BzbS#f?J1I3B!xr)HHY>Q5y#9%W86>(tErC1nP zQ2Movy@FL^J&kztN|kq;HpeUzSTUkl*6VgUn^cB=w`xg4CYc0A zG`*@?nX)X1!nq`{BCvk;xr=s)JcPlg17j4$3Z+b@Ee+iQ6Pr?wCq?Q9yG-L$Ik51P zG82;aBTCoE$RyyMq|4fa6VnG6?iyeX;rG-DFZ!O1K9-X+IMnqv4@9obi8i_DaxB}F z)L@WJ{rEG2azz?g71|(+^S;pE;<&<@`-y1>2D=7W{Scn3mTAxXp0)8@qe$eo7v1PX z!|vJUu|ucTuC!w+@oZ}dsbq9Ad4M0%xVomEe$C)CDc1)(p(BrO&ge1uc z<5KH20T=dM;tbcLwyQtdn-^xJ#}0{2>A+0$Opkhz9^6Rc>c#_6>f28l4n$nmQcP;K}9Uyc}aAjJDCtVW>YXcDrZkT+3aq;#2TS7ogj`Thd{%noCpw#t= zf8z7)rN->tLSzl*R4o`C>pH3!|`FujgE#?sX)snVHwoiCmnf9Pdw18uG zd<)@P-h93I`tkkS*ROecYq)oy4z8zK=1yhD>~}qM`gxfNMn?6q!ozkoABE9hX>(`He@ZdrYD~~t#ZAehi*ctINDOp2{znb_S zuhpJ;1}F5<>U(mwb8?3N#BCOU zaG@{w>!VkzuvgmNgH_=}x`iz4mG7oc!^ew1e*%nh@F^3?DKXc^0^-Mh=o2ke18Y}jYwEak6!-$Li*JXTC?cQhdv z2P^Dt<&rq8opsGi0!zjs@UR;qp{!*M6?PLDwLGCH zux_rR9I+@lLruO+Y;0~O@0J#As{>XEMt*uy=P^YjoCCJqXsX#m1uPf%lqzHVsZ?rt zc5XMY6vWA#d{=%2#pBt#!Z4HFq?IXeV(3;{Rqg_-cc?PL&extgVC~?7v2V2NBjLDj zE+FG5?gv;6R3ih7Rkp)Q;uYc&`NR5xY}zt0&Bbt4ctiB=H#nL4Q5$_`xMVSntynA; z#8~S{X-6Hhj&SMNGdA$F=O>PY)^Uwy_EMFFK@HLfGMPxE@h;cArEQYs2?t34ji+mC zh!Fi`MII22ch&w$mM;+FGFu>u_>mt>DxKC*h<~Nw2S2Ci@A06blv+w!OvCI<({&}< z%s!Jnrr@zlhr~tlADLO6bpyf?F~@+CdHc*JAOb5U9%ZUa8xXXw1q%6?YG9d6A4=2r}QiN}z%vN3KN~O9I^;@|Ty6N{9%=_N_yve<<_f2lv#yL=-v{YN4 zp7XrVdCoZlx@?Hd)z^dg>Mbph7powL!jsaHZXdRQIpB3v2I5uf3^73Um8{4dSgt=f!M zn%<67T_3-Y&80TgY^c``<%_xKQi6fY0I>0y{DDHy1z4Y<_0j)JCH%w#gVPm?300Y3 z7=-=aL9J#O6X8r(#n7@a;PJW5oa!57sB({%e9|JLoUX2>b8E5=#^Cs5O9xzGk_AtS z+R|-}Ef1UPd26Y9CG@lKaW8|bJz(6b))uO%ozmTFy2LVz7mupbw;oof({fV`z{bfT z$}cepMx)WGrK0hOYE#w*!tC>gXLyRU`JS|gv$dXims%wZI{-MjN`U&@b0TIAA3ijcqm|B0J__Ql)f1nOERb49vD=zl`aM=*?~J1$mpRPX%sOy z?$9$atd@>U2Lux3``k*>)x)B6z5QOq1STTmXGr8imev7LhnA3;(+{LwyRTZSc`}ZPXfP!1a~_i*iMXPse7V>~TuWoJHjQfW=eR-Zz78$@WSQ ztQDC3cECEkOxNf9VIyoCDdGtDo{|dWrG30PYJe+~K(bbtynLYx9R+dORPa-;RA77f zh8YV%D^!n$8e9o!fOY+`q>?#lx%gYMm7>D%k^WRSdgB)M&qRtzUIoBHq(A*Ffz@%< zSBK*~FJRtC6%!@aVkd3J3hUl1%y4t{FC#3%c=RA|BxZ{hT6$n1Tm??QDX_50-_%@p zmGF!TS9_%@q>Yw~*P>b2zE863cp$haarXGZBEDu)REonKEwH`=u23}`sk`leO>~tI zv3FZ3d8egJ9Q5c8EDu2(9fE)~3C{pT&~vc6c%0e-V^sGt58i(dTwzPi&kyf(CA{u0 zebf)%zM`)rXx4&gmM}Osf!0T*?lI#*FL!>wM6IQBmd-HW1y>!1F?i5?mn-3Una(b7 z0%Zb{MpjLFz6dSA43LC|8OaZ7ZSP`Am&1uvm2hTslm#t4oXyh4^l7zd+zQoIYlrqt zh^V`-l!`OJ)7cfAEC?QMS+9C%@`}KCN$*8GM{&R~=Jp<6xz!?ZLE&t`vZ@APZ9?bq zX|eY&4WV?E&`QJe$8B|kEv$}GQg<0gHob9zzyn7$?P>)^#~Tzr5x`EG4qArs6|6le zI->0W^c@2(bB*fYit5D{wP0fLCN+mvT9q*3}Ah10u>O27&n zYU%Fvr52`V1H~QK+4W=yHmWk3Zs8q)b+W*_xs_5K?az5l-F}B;(zRRPN(bQtifV{^ zI%X3-H4f~4b+~f#Aoy-ed~UoS6$F*6Y8}8`!|{mR2Y3-w8bl1jl?KtN4?Y-htgSCB zq*5PJsf9KC(^LR94y$AE8}+il`yNbGLfzk5r4wO%moEz^x|NQ($wIT&0i3H!T-mjJ zUbD+bSit;N8U}%>-{c2%6l@QPzT0!oRsfQQT)}FC)eh8D3*32cZgM9fu)QXG|7Q2A{Qbl(Nqj)jMoWjDtTY1uI45sr? zlEqqT45}pXr=8GL5B&Y7R@kMZ?)`%{M60;+l?mv3toZ_twIF7qH)Z9nzaE6Eus$!# zaJw0$tw_m2qWQ3spZQb``_-*dE@sN_YmCtatmA za#OFM*$rbWGRgD6SSx%SnjE^}N)mcqEF`AlOG}BB6&5l5%-M}VqV(h(ic=B`nq4!f z#Iz2*>%&bv$aB{!;SEYwHno}QNmgsq&{KS#)yjYo4!XmH8Hz&4vMAy;OA( zcc#>FWk^Cmk4b#|K6rAcDj}X)B@xDUgmJ=wX$lfLToCh#lo!q zt?csfBH9&+EEV??7ZcK@aD_OAtbnBw>wQ&9q=j24tv_ zVr3)trCBL2nfHrnQ`lmcme?rE_nzA03a@;KNSln}b2(TsFa~`a$Dd2J&ecumOpgytfi&p<;%y9@1baGYkP73{{2Ufq>uL> zKWk{L@J)#F)daA9t4O|l;$^^NE)hgQa14+i`rN+B0lc-h-R;Pz2v*45!SEv}+g#>J z9)0qnQ6GD4A!X`uIh_s%By%m9orTngWH7i^5Ym%lxyMLoBml`BgN9hsXV=4OO_#Ji zl>Q67mX3mVcD}<0PZ`h>tvH@iSHit_Fh7a@$z7ss?We-yP~N~Bx~-AeBQOkbSy{@A z02Xdv2F9B0*0{Pd>IB{a=WyqtXsIo*hI6NwWC=%>eh_IwSeu@W;mSCx30GLmy3*xk z80xH@{(c|+{^iRb(DyeK|M)j-pMO0^v5n$PiW?k9C?+N*KF-h2qmL6e+edkPQC7lp zCZ{H#Y(iQDf*yxrvEbPw-eQYK-^6ZI>9Xopj~)0)kJ23e8v(dCIUkj)w!pF?QWcCu zMn8~CTLw;R3#oO3R$yJ~Bxcpl=+FlL{qp5YC;a-$t5?53KLu67vAcUsMXzoO&b%Iu zJpCN*26kEQGBuH<_^tu5aC|9j$5%&`cwE?%-H~j@e$bj$LJy1@Y7t3YNERk_O*NAo z9!r;W|9B05{_CrM!jZ9-f$7yi5ouz05@|AZ$rdalrcGMpjZos20ncn>)W%Q!tYo98 zHO1<6+@eWU93LR2&v-=*y;&{rpUm-Z`1RGlD2E~_<7+>N1wGR0;Srr9+L8SXbmhTf zVaj)j$SPqON``s0O(UOpESUa}w`+NcE6w8c^d#LqouoTWGQu^#h0t)h+w5%%2 zN0pkYj1N*3h>nFIxDc|jC=@XWI%o`riJPFE5dvn>WM_ogX*Ru(BsN(I>0~1e1bSos zg1O(V(t3Db->vV<0TUCcOG5qXJbveQ&T;cuEK>X!bfw%fZZB}bl~NWmnxcNokAD9v z{`=?W&x*%d%nBVi;fSjprP8YwJ~@0Q>(d5g_=+#N=_MtF!6;8sqn+c_rPbpyVuL;` zbEFtZ&c1$wkKF^K&5~=KX7GZ-$x6CqZ zYTP=BKsI}8BLOk!cBvbKjE5aUpcBmud*Ev0mtQqwVVxVU6p!@>BVfJgE2`rz+JTBKQGCS_SkX34 zOhZvR)BS=Su =Yhq-RE7X+40%53)wJGxR=UW&2Ou}Y)JO5ZP|ql7CVLv*rueay&P zB~JFJ_dlxPP_|PV2B_99C&Qv=pal7JTlw^l&pbZH;}*~tB;squ<~gNUY!`Vy1BW%; z5)Mv*C5!^2E!f;+M2X^)Enq2?dh-_btJxtphA;Dm6j+7S13Esp!AhfsyI5&zYTD)jto3`q ztu>~=`h&x1u)mZeq3W_`h0G{g85Di`oKC=fz>}Z$G8=}-iI7xTq$MYj~w{X z@d{v#Nwb2GT?Fbb4)@WT5ypDZ-Br$Dgdupo(yY0~?Qrd6vw)&&my=wb!1gxOj0>kR7dM!yDmQuYB58!q+o ztkYAk=q1Ar7tO4^?y*AyQP&{kn#?clA=nC6(YftD-@nxTB>z)@wX(YUylOzFZd6UK zFL1cPvxe)}*z=opz2gL+|Si#`;3sTzeIJ#Iq(n2sZ)9VHzp7v%ER zw-m6nv2oaRW_jFaNHY-Qa5L<;H>^cj-(AF*pkdjm7W-7`{#~fUWT&-eONk_^Xythx zlVNigleku*DBKMUJeEEqzpNXchug@$fV88OywIx#tfR)X_X6$ zsmm|jSgiuqN*?_+D!sY+`FT5FdpUp-6emSPF<_5R+;K4sqiBzyj8qrmQ9C9tN1 zEV>P9JStVz7U*uw8C;-FX%`OW>ikG2nIsCj@w^sTch+CtSSLov0z!bbCTxiS*Y9w= zd%t!I(iMjEXiruLSlRK8R+VBVtr@BcakDs@Q5#x&(Cv>_14}aCjHtv1H-P|67bvE` ztkS7HV$p1cD;ZReSLDxmtdjlX#>)ldx3{-9R&7!8Pr(sbq_OkEx9lR}qM&aFqa7U& zq}Em>sC?Fumdg@^%%TK2=_mtBkYJ!-r`ojVNJ6A>abbhvAgPt5PatF|)J8q14TalC zBqZf4K^An0SSo@|Ua-TB=6D`Cd<$G+rOWyO1;NBJ*;zW~;b;?@E=!4rgn8&KIRK>? z?S}4~HEfvDXaz8ui5sU8>E*1IgE=BMl_+OXAfFOzL*e$ysppa*CGy3eZrWKEav!&T z;|Q$A--lS0hJQm#Rxq_0D&pZC@MK2ssf7+D)w|~m39yF4h*8vePFbYzaLa~)wscFynJQjgf*1<4B`rTap1`3F zuuulps-41c5*Xs@vet%hfeEsHGLM8Rqh&D^?unTGgeNAUbynp|J2HYWWB@F3{DemC z7o^e2Ck+Fe8~%xOJ5k6qcg7=8Dp9HDwEukGJ{~GofwF#TD(f~FD@V0%*B~?@`M}7Ri;*5ffmM<0lj>sn2Y?NOibM8Um|m0UN_Yq}8qitV+^! zWg~$}8=FAhHgdKX&sZSq`zpzdJ`j*NxqFu^i+pFUX@E3h0jlFY2U!9v*=+!=0nz$U zg_4d(Y22I9gZ*M4;0t=v4K~c7q*zNI&49H;$W(gn{We}Sa<_*J2BRqL;5vR*Ibx_ti*;Xu=@LLRE4*(*7yaG)mQgT ziSF)m)(eO;T94fbO=W5sw)<=US3tY@jynaR-`d=H=tIr1;jA6Id|; z)+THTOYh;~vs<}`oPqVeZeV?dQEPAi*x1xa#=JV*5iy<-s4V+H!Uux@7Xzv4Nnhwf+i8-dFR9tFa=iLUbC>Mnt8?D06bY^JSvxS#+qEA0!LWXl5iTcB zTjVgSKS}z%!Dv;^M0LpC!*#k|sUysQwYr+H1FZGQ`Gq+~x4#V5WnHa55{@tg)()i6 zacW#;JB)$VuoXV@SCN0@m9i`w*jkumxwS!?B)$4& zYC=oE+B?}3$sW@JEwBz?coma?NJL1j`?s1^S&9lVS2eldr;=~=TI~DTw5@4Nz#`1+ z+)Mg23puVFY2&YTw{>VEp^{^X?{xH8Cu;QHtQsY=WMuE@uCZD)?SrT|q)xfKYtmhl zBrVg_gu@}KnQ2OMPXiW!b-1#!u?qA*r6%7Wa;6;m>Ya-I_PR7e8MbC6KH(WQ{P`1pF!!=!C` zlAhD1Nl$~4@-A47tQmVezT>;Gjm^POE{feMAW$NzEUd5ymTQX4iAo}K5lHEVVj+%@ zV1dxt;H;BwvWSv~bQOt&O{7RH^AFDVjcpi@$IQ&<8J}+vMG+XOJbf?E`@GMaT$Bx3 z*hZ2L3#lUE>uw}v9>nyyh?E+1;p0vEJv%6c6?z|b01Wp_sv5(?@AQsXO*+qfO47rn z($CkVmRX;cwSYt+s~ofvXKNS0EC@aQ@yfyR!FbGS!D&%i%FF|h70adUdg7}6A?&C? zR_8j3^lFwZW_6YN;e8!2Tx@YBtolhBC<)o$ezi3Wl6+?J)|s~p+l{Wq6J{UKY#>0> zA&Kb<6?O=2$ngioXkD2G{w|AH>34R}+E(QpqQ|;w4X&2q^2L^C!m7VRnFf2=PY`UKo_gNvcpaNN)3kJiS3Sdp>S-kHJz~yr- z?g<6J+OO*kpOpV)|9D!o|ElcGfP=iAE?_F2GEgzmM~s;GUU9?%9&eSPw;0=Xe@44l zLn=p?sz`=&0KNK291eQn(id(2pG=6IO6!dsK0@)Yr)AP%l?ba%oQS~2*MTsLO|I-_ z?!^CP&{Oe@$9D?@Ec$DRqFQkPy1oBY&!_Aa6(_^eM*a@KvQFIlp#lDJq1EiDfG1s( z0v5Ic%)m@`b(9ELljOh0ZWORgC~#F$V!1u^qr%-*$*v3*Sh*5Lme@mHXJ9QDC>139 z{a%Rb5wup8A@X%wtLKEZy7f9>Ju!f@3t%io_K`u@Bo9~~u*UgF5>!_$|&#*K*3=;?1;_A;~xT1l>?N#zjm_)xd ziMH@f*QsVEl};+9#YU59gvKa@h)ZSBN<0c+;0j2H9T*-L;=wb)j@LR4c{6!6oMd(lrbuy$*4e&+dn(5aMiNb_ZpGZ+ElZjA$5Z8zwf zl&;W26zy*fu!htF3nxQ$18Gfz*6RU?Ty5PxVM7Icm9*-)dA$GegnD=~4F;^jqm-*i zb3jp#c#BzLw!sZ{WuR5jXJ2daUkG8tmxLA#)J7=cULTo5>WvtCxIo+;whisNJK zO$3m6QK8+RzWMlOzekOVbP@2?RR3h0BQI!7%8aoX7xcFXSY}n!`-$@(rBy?}go=l1Wi2 zm&<0;=`_&I3s~F`SMp_cksNtQH|Y%1!yDV`p9~SS@)%T+J9qLo2vrX&%NkYf{QIn- z+nN$kIAOFloxKnm&A{5KEND~7YRG%n-XdU$OLrD*HnUNu+C9@r^3-;Q`8$4#kQmY6aCKh8gL34;hO$7p#z-U?p`Hk4(eJT~QA8eCk2L!| zy5;a;jCt@y2|?*@w$gyz_w-q*V1ar-1T2zdDAdBM_0Bq&fN!r}&<0=O=QX`1_qPbKu%TR2IY(<0FGb8cR0}M=Ls^q2Sz81wrW=9CGH3;Qs78+( z=f9EF7+603mR!FZ$Wq}B7=A2~;i^yZh+$_C$i+Ud_2?^Sd~X}wIWm|fufZs z?zN*o`)a=-R$<6ZlCo8!q$^0s3^MzO2^;Ke!*qgzBSK&`b?tZVw^^m@_j_9apIy)v zSEzK4!2CGzsPKADLl;{8L(u+ZV*p9yF>-aZ&$xiC*G_)&Ro zDPUO9oUvxOc(Ph;uU`Zk3G;`SOb)x5jfGFoU54Csiy0EiQCQjms*7M$s0FZ4ZwSil zXtd-+h4bfM@Lii6Ij|(gL8FWKRL;C|U@4)&y{gV(8d2*<75ez#WnwvB06c|F8~VHG z{H6okRWyZMwTrI|zPN;d=%P=epxw%`EXTSjpaXD*YOJqdo<_JjTF3xo*45SQ+e_<5 zaM!l1Yio++{Osm@ZFNwCc@pw=3tn!x@{-JE6VJP@tq~(<;-SF zZ1ObJF@2i}{zqoEv6bJ41y#KJ(GGv8VQE1NU`@fWY{**Kf^W~YwdoMB?q2)%-JkCg z7g{;g14hRr7}m7%F;NzMMnu-oiW71K{=pc_CBg-NV|kqifyP4t(;nhDH#ic1&`?LN z<6B(;0hcLI!Fx^M5nr(%!9SHlKR8rAEWpI7cgcGmqJOtMwA`HL!<*s3_5vX*O6T)G zU|8-ne<$p|Jg^QX;PX#(z|}V>U|mD+c7${eo#ee~n;|T)Ef)F^nKCP8qEFVC{RtaY z`Gxu#7xC71udRAf%ui7x773vYs<14AkMNu}hG7Zdkad`n$%QMs+H&ou?>zj1V0G^{z#{1!dUSjghEvy@dr_2h z&K}kI@qcPNjvz+Naq`uUy|MYnB){^?^v9|lpM6u}^dL%(H9isH>IdS53s!C!h@w#a zRO$F$tuG;jtuHLRI_hlVtyd2Tls4|?zu#yQSRJ#_`%;E$*O_PyG8)YOiBD&KKVXMeJk_}inT6Aj30oD;*>-p_K$qASJ z_6!t1bZYySh^{p?mMq^8aaqq+_Ff;nJva?lU$p7+&cUx3SJwvMIe9Wm&XW%tQ#tG_ z0uckSUcX$#GU1Iu2Y}6cf`dk4J*L(**+vxz>kB%nMBM`kum3eMUe(xB7bL>_y z0=QCpH?T(`FB>YYMC# z0zreE&k<~$=p43S_{P-K6y{Cwy2KTo!WBP%zRd&cCqeD0)^sKd&g?i@jg^Z)l$UFz z##d*?hWXg_*c@(vF3d@C?gAr#n!E94jH8xQD8$5HOiP?JWV$)be3>Y>GEO4_#Ul-I zwGH%!wz%pj%|@!eA$u=}dCGMTCJuKeG#?F$G|B(rf%V_t2ZGV4(}{k!i|Y5kBG-v< zl}6!8czZQ9o|_M1owI?qSmX5}BLq~;SX)QYCP$+&UA3$)a2&^?Xyt}*Z6a%Q&(#hP zXA~~*FgG;~Sw;BaUwMs@g>V$bA!~0L<-w`Bvfl%GS3c1XR=9S*#{;YU=e2vkk}NM9 zKrIxgDD1%1p_rj97naVA4lbx+aSGULzlEHmqgSK#luIl~R%3X7Z$aHO}5!=rU z#+1vHNtFxI71mN0z>ivotkM+he)oDAC4>B2#ca73Cce@!TwxY#;F|E=B`KCt@l+k4zvekQqdJ+8a>yy~Ia#hdB+gO$zWN_sq z7zIvRR9!S|lyND;D{Y%XO6y$6+rdu&4CgmGO6BDEKkBZgDXlGwl9zNRc3yYa>+U?u zRQ4bv>J}&n@*yZ(5s(+r7_30Lf<*x(BXFWAL|*xLg{DLYR%tPSlocm9bY%a4H)Y~4 zcncZ6k3f!{8dZItN(kkcv65r0C-N0m#tZ zVIDfO(=LkHp?A7M!SqrOT_*2@b4+kY;6fLp0_oemH72t?Sv24^g4^rpOQA5X9avvg zdiX>$Zox<#S8q-jg7Zs%^94$Jc=keHTp6BrxqJUqV))(YqEZ#BO+jdzfgvS3zY4in!QoO~Owz6_@~UM$EUg-WFM6fE8t~yFRec2HxpFAUckoi53fq zfz>J4C9984k`>+un$jQPr3v%{rqUS_K{_VpMLl#0izACChl!#(7k(k^q*4q}V{dQ^?b^KI+)Q;ZH!X z4FIdsF@9{}aCb!%YfG{6no+xfq+^a6vj&ztU^Sbq;EkGxm+dCh+_83e6}x{k&o+0d zJ#&OB)~=4g+BWLh!r`0zLZ^ISty95j8A5Rkilw9j9d{|7w%%9Ienz8Rk7wk&y*DB; z`o|K|G5vO{XP!fk93(0hQD9w9YIV6wg&a};7`h`|4ZY&44IOJ%dSo+*;e%lyQ&a*! z#gT!Dkg`}0k)(t?%oZO)Lj$hL~u7mfaL-V z`wUZIH8nrdS%X8YJ~m6a4kxhC4jZI<{n? z6Ta(I17{DsT2qTwA$&y`SURW;sD<kNhE!zP~eG0qJ1Z;|B2BTnLM?mvcNVrq@wg#U_w{TwlY`E*w z8>A38r$M}h0x2pMmn32M!xn8&n<|8&ia6rus8IKD3cTO7lJ^a`Bn_;0kFtfs_socK zcjWkU?QdZeEooA=qGGnJ1n{T2fPX)C1{GPF1u`E<=vZF~k5arnAn70d?bDsLaJc91 z1D=`Nvsp!cynM2Fs7xi;k|Je!HR}o5s`LH5{5x*hB+z|~64)Q$APuY*IlbX7e4=9V zs3^rk0(>P~g_R7zN>ViHxH~PDNF@_(DUy{@6Uw}!np=+zOyAh}zn9-s+U*6-`T|=b)LAMUiOg zAfM>d7x#p#7eQ)sMDC>586F>F_W~20Dh1Wk^^r-;q!BiDBrTfb~ScJu0F-(5znI7Q|>5 zof~*W3JRdfHVuMTseH-^m?)t358(*YP3ntI2wR&2;I&)k#*yF8GPM^Wv4r9e9e`Dv zR3E|35*eNw?(IC25E};M(tZay?xN0xE=>VJHGY^0ip~zoF?Z&!4eq3nn-n8^-TWco zH=;Qlvq~k2Mbs!tE(JxYrW+H*86-;)!)kB(7Pjp(w^)NR~@O*~ww-)(|rM|6mr#?}YVgsjSg;0PMQni2<>RWFHp_p>sx^$ z5q-vmu0XYUTX`C&5=(WE*D+^po$>jAY;+s)%8=Y1x3Izdxh>LN9iWcYm#aJUiK-PF zIQ%_xv+S}_7JNybvqF#<2!@zqr8YV;zE43Y&#`9NWop&}ipExes^QM&@hOreiz$^r zH7{$mTC>?|5-M1xZ4H{}i+je^i&w$)2+m2bW$I-y`&OY*tyVLOol{>c#PHl)9Cqb` z;1(N{iDD3rB*iSB7c7e3c;f_oqdXV%?YGqKB~x&|ecRpLJ-a!(yVq*;234=Zi@TYf z$BmgGdg^L7Orw0i4F3wvyQKXs9{$>T>4%(~FtA1^HoTv0wyz%R285gFChLjE=m%`W zR`Uytov77BpWi#KaKr}e!!WXv8iJK*qkKOhGndpg7r;~J$6$F_r{nVgoP7b~{ObJF z)LtYKSqeZHZ+SqIsaNk#RBECx4G-Y_6V7!5Rsj*MQ0PzvX&2gYafq6R;snctCrj~6 z%VbY<5@3TJ?~OTaan%aL_Qm7VMA>44Yh5>BWgCl<%|nsoWeVuLa5s8Zs$GXknjEm8 zAF$ADzt_jHsfKa<4_B9GYDS?uKGC6vfGfk#qJv*#PCC5^`PZ419i@)sBA>Lgd|6`> z)Z7XVA11J-bOe?M!tmM0`QyUMAB#=&{rIb@a`iXE!#H?igPMbY)fn|F4-`^f@xl$oEr~wv|=@rdT zBudE9C450L&qT?oXd7+;>IFy=KOKt~oV~rg*{aiDUog%*4Qq-zKi5sZ=GhNr1 zvwK@f14aN!p2J@wKps8ja-~IV*sPpq%elrP9b;?lu4=tpF6Y*`UG4Km3}Uj2RK!U; zb3@W_G(SodCjk-}GXjHP6X8kD;^!8td5%_)wG`mW=yrd4^LWeEjkDogE1Y7X#YV22 zEv(fSUHzY=>uIjt27)prD<2Xg?H!>(#^<**hdXNHB<7jnaQ2QEddcbdeZJMVoFF@N z!QkrS$?-9Mxq3*jLe2&SVncefQdr}|iSEGZoO}JHna}0Q<$8$uK%E8I%A(8wG%8ms zDT#4Jr&;Fu!E~5L$Tp$7=fgKRyRNm^TIf}nbw^KcK)<(hRcjuc)r22}Sl-c-2Uo6S z#fHs7cCB8exo~PQ)v1zBv)5P%L~B)S;-JH8}OwIrZDRD2<%>MO)|fL0W1{B@UY62;dV0AV`i=$G0fyLH?9nnvoAo)>S-)Pz)K*R=S_TRp{5VNOzAh_}ni^ZH1OB zSMg*rz8{1mX)nwIZa%G3bHI#QcMmSL0E;PC08bto zuKLb~bR}D61Ik6yEI?^El^lbu5Jk~Dah2t&)cFfprA77V7$(D#SZ}DgWDV#EVXhG3 zL!Lme&zpyRT`0}Vzpi`RESEZl|nTW6Bpj| zv^2YFgUMu+p-)i&Ki;g(ls;MmVp#(bY}a94w|fC>;ogY$=Z~dRUYw2kt%ffWJCmqyzbZ0VSCF)~ovaw2j z+YnIm;tu#?)3YG#9=AsNLuBwgJars8VR0E9|98PbAg5tkMl}gdeBV8As$4$^G8WRFB38R8%?& z?^vkNSd!WHtFxTKWd<-7Dy5$(p#Jk}3W&NOk&koj~NQ7f33 z{PoKUg;pIu2G7NYAINt$2n803F$P`vPKbQ0{t+;f_)pE;@6TCeGH+RPU?>ZtLkQ3! zsxku6b6};|(ptQPu|7W|WmB)-iI@so5uZK5ma$(e(bacN&GihdI~P@A-96Zj`SJ%Y zh#rG^-JK$lK21EfJ!;Cb58{n1jLt?gc~1|MXL(??vT3z1>Eiv5{r&wHpa1ds&u31E z5I-!Gwia;G72ePP`nlZNTR>Mzker(X{imtG#oH?m|?Bf69VfM~o zAz6*!oohgm4TkoC+`<`ImQIt}TsT>6G+|{S2c)zSjLF1V0j?0%Sf?VqE{(>8Y0#@v zYh9cLexDh5Qok*op zVa!j-WIpqyn5ZIo7S_V*Du30ebd{NGn7(_FVSst|&LHE!w936n{#CT7GBX5TcMpew z7%zH!h4QdTB#lvNwKz*ub@|sOT1zRbVd&}t`#E`FQJn5Q;YAHy3NDX4{D$k30kl63bqZY->%4w6x33DlH>Y+~uvbu05yV+?uqKP)tAC69+NgAOfnvkL zL=2rZf(_T)g4N?tfR_XeiD5$iNkR8IGSlShol@MDg5#d z+lzu>{#t1yy23V!ix3+!sG>ZxyLW@&;IWJQ6ZbDmRbNE_LWfvDKVv?R_U={$t&{*@9S=gUsKDyWwx1i1uISU3 z;))G325>M|cCT^h_aj(F@ZHj1FI3~isLm3KFde#u+8-T zln2)1rOMjY67x5$OuWZI}_2uk0ly+^Q8ehRS$w zDys3rK0aHGyJaMc{q({pD?KQZK|d4k;HRg*e8KybLTnV2Ped=_+^Vn4(_q2?BgH=Z zp$(AdJkov!?)>9?DlD&@NM$k!s@`!c+BqO*D*U(X%bnY;V;e=#Q8v-8IvA&MXke{T z0EV^2ML$Z4d{6K1J7Iqz}ZiV zZG+i!P7Xu1(}L=*A^IW{#l3UV)dpnsz7vjx0V{3f?%p7s$5P8wx9&6~tVEXGm%&g$ z%9t+MlQj=As#m+YnLj%-c5h>3c<97IMmN5nRFgLDU^}{iPF2HluR8R;q9eXFpK~k@wqJjfUH5>4c8|^RDrU)Xm^?BwxCLyom&ORFmve2Uhs(Pko0L}nHk}AUhAuLpeDhiR@kHPJQ?=a zUUN5t>}GSr;;Q+EXokWAE7+?|O{@ZQ$N@X(O$@cs;`A@>@6FhV*mlLV=?JAg*)+91= zSR$1^rhpe?sR6$Ew&~lypijQ8E|%2IwEM!A9ULSxsoNP@VW|*Uv04Dj3_3G_ndr~z z;ZR!3NZ~npKA*G(Wb|}s^L4o1;B-|q?S8Pl726e#s|g3#r-`~m!GoEZne9Sg*wx*ZyCU(Kv z1D?I|aCvB=BHxi+(47I6sTPdoBJc7V1sCt>ynh;(?k1kE*uOVjnz%;UF|{lNgPH%9 zfY83-!ZcwPEXs$Dv3v6;V>JU-3pwr8LdQx#prb=6aocP*w|l*hmke)_!?jC;tLCpV zKtiMes?*cc_$gQqgw$L!4xl_7J4D;y7#Uw2#)ns}G_X`hf;?3N%I+39`4N+N4Rov& zuW2!+(9j0AE)QWfeVtLpW8hQ;?xXmC+bFz;Xm(>9vmo6ww9@y-A1_jr0T8$_feAXJ z`>wMSKt@ayP8duG2bR$JhMl%Lxi?=U#`*zvXD1GVG5C0&j<+99o#2_p!MQU%x9kJM zEaw|L_^`bnPsV_6Zf45|GpI;A2bOp9(>orcaR6<^YJD$ErUh#Zf`K)qg6r2V5n=rm z9N9B*Szw`vW9=A~pj_px1N4!$xubCb>`P;dC;knHP2*o41n+uaUE=M|VuVuj0Ty{` z(h1CjgOkxzT4*ZlL%3?bM7V1DUIh?|2hfjibHMr-kE3cU0uw<*K3l&(^u(+WA3n6Q zj6ejp*%~_C}D-gL~Ax;B8 z&0%1D`1tX|hqoW$_i9;?tbtRmrQ@K-kNx-jj>6nK;Q-k5;)%Z)?heT|$#-M@pS-hs zX*18_cshM=r#qdQ?v@t58gdh!)vAp%W}ep8tVUa@4Qj*QIFX65xe3Wdl4Zk&K)@G< zUUfsFkq*;%6=XqbAv3jWgzA8>-8(PJToe_&^VUCLf6tSsNuK1%^Lt*P`{Yn>`a&-9 z={e_j&i9*LPv zAYI1%Bb>2KgI>_8VRgA&dwYA+2L}iH1v@+>zsS#lU3dPluhg}d#NQ~fbD zBz``G%TvJCJFP;3ZGWy|`*xhs8&`&_n5_-ghvFB>gY^xn=U6ftIOS0^lkqrg{Z>5V z3Rx8@<}zF(Zt{2I@nyfGVg|J@XFm!ugsM)O(yHR|9KM7#4p<%*1ceX7W0qM9Zdamx zs|C}wxiaX^z7mOqrVo~}v2a_PnSV4|!d1cIWDF@4hFt@&{o_A2q&)t*zAsX!o<16W zSXga+&kWcWywO0En({GjGazwWLkO;H z#$xMEVD@F%BZ1kk$`$ho-x*XQ#hRr6%U&f4(l>BwBx-?Z^k`T$RF6S^COUzBpsT&( zqvhmP5we0tR?tlx%}SF>w!&-P2a1xx{A(7>os(+;IqcW)S9;{ia2D?+gJl9EkTD>o z;-G72l;^6yo-`s_*m#vdz{~BOWWFhRt~#ii*F7NqMLpdp@3w+R%C0;CjqQ9Q3YK2E zI?uFhNe9b}P-PTUcUhaq>y3K7bt9{{u;(KI`)Jhfb&&AsDAC{0Bw4K;1Y}i3$t$s) z($tH=Kyk3~BMyXI9Wc16N3IN)DI%fCG=PXa8AC|j`TZ1n9`xSlax88o!~(Z0w-P$Z zHVkIFNjuvP0%5)){{VYJuAnEuXsLZp6A4XiNDmH@hkb@p#t^bis0rB8oFBB6zXE7R)g>ekj)7;!0c zs3n;bPotHgf>C@c65p}u>Jlu4$SgRhz`6~s9TqXZ8)y||SbHY}ISn2Oj{{-=4vGd& zB#pVXgGwT^~27;Z<6y_gJk~4zBM|asS6dH|WaUr3mJ=&9F%F4%fbr zt1F-BmMg<)Ig!xR0xQk>zXkY=CAQ=nfQYV(uDjS(0anC8x%O{_Bw4<7jnRsY1AFD6 z(1mlC^vl)7D~k}qTA3*Nzymn@f;DbrfgIveuZFVq0LVDjNc!R`|XsyLX|dx z)OA?Tj=B0u=UkCU_<J6$I+6@^&K5Gx=VHM^r;HVv7|f*30I%%pDUX2obnut zDnhwY(kVP~SyLhLRv91X`&RvpJ6Ex*RHEdP#aXmp z_~M2DtTiC*5eb8UFG+RMz?+&X1B(Pis=bnJ8mj$BignmPTp7;Fii89#x=6TO-hw_Z zzKaAy!d23^gfvBx#qD^=DcDUltMZoj(q;|{bE^CH|I-txly=@MM~cN2-s!;dAXo*g z>c}Tl=;bviF!#;yEhDLB$m*qu5(|;+! z3J7}8yluST=2H20xGjDoRJQ9h0s__=(q5_Qd&n>GjH<}m{-AcL`?QY72Zbg}l+1f9 zy4RYpRao>{$+;I`*%{v=Sbqv^D&uygrwaMo1ZKFGBQW8XSZc5*6z%T5-iP&kY)2EY z#<^b3zFx-%1w|}*qGV!Y6ku_;OeTcYh7&odF-v?`kiWtPCJplA^>QPt454v1Imb%u z9l0?${P^8pDz%jdPIHMsOU?p(FlV9^*XkKKVqpU)TEtp}EIiKA#E zncYYhY>rrx0<0U}g(-)Pa?MQT8o(YNPPLncc_;P{!2GFExRP6JFq!T`!r)gR;PG?! z>6eUjmYtiNBq(kiH>*7~bq(`VC27`^vG4$^I#@9<(|R3uuAX8+UW#Q3O98780|&56 z36^mjX2xvxC6i0@Yjj}w=>`flybN8QVv(1xZpWR~jD!TNIxn@j-|x4A#K4*48ZB4G zC5-w5g`vSoqj7*jw-v>=FO_n2eOC1Ze#dA3^6SRoN`53XwTXV&eh+yDa<*ZVXXEfh z$ipyYa;dSnGMtuMcP|KM9HE-S42$y$xyt&$cNf=+Ww0dU zL@JUA1Vq)(_^tW56Ze#Yb?^2V`B-+`sX|XW@6-9Xa}_&IL^*h4~fm)4vHKzKuov^Lg|0w{X5`Zf<`5>NV`%nq`iN4L>(G zzfoRzFPR4$4+fQ_S$X2Gt~Hv!qB=%VE6w_$yma|L_RcP}sWgq_I@Qr}{Qel(i~pvW zi*RUKwRP3kR$a9lDx-t0$Ez|2*Nq?qw@^DENG~nRUM0{?DjPC}7LpLU#K0tN5{2mm zdg0vJQoImL@Aj&9z25gt(oWLm20Q zYp`t0hC}sGTqshYnv$^E$Dw8fMOj7BL~yIHOi-yc+VfAOIeI?aAp`4jU*GAn3lkDt z{ojw#SFt^^J$~hCR#z#Te){C)g@W4d9}mo#Ot8i=Q-mnU|KMh@Y_`0L3*Y~v%ChE% zJax}21Tt2|NqPAjJFmm7FqT@a06lohv@JRCx8FT`@{$#xikW4qlwp;?imSO1Pf=NjN%suuelAePeNDL>=v>;Of*MhUus(N)4fmO_yc8aUXYDPfD_^oBOx4 z!EMR<>O01>tWgEKcC@BREgr2F3d0dfuG(`@-3gm+OYns2C3jlSE}jT<8?l~R!`f^~-)tRMSEjAZYh-h%CH zU9M=vh9&CEedB_YW{fPGEt4#GxfERYsL67=@Wx2H=!s*k%a!DnldUUfn&7a6&(cJn zf{eL=!P`z^vPSfW!qyv`XolD@3pUpju-ao#<-qcVG=N>F1xx=nWzmNGMJsb9xzTpZ zlh98mUj+iWTrRR9X1Q&WCgcagmaM{=9j5cZeJi%NH$iNeRR|rHr_g)Mh69nnvRDJ6 zd1cA6d1rugq?y(weVwEhSJHd@D!BTA&|yUwe5sUMRF;8QL|?hEUZzD_GQ5rOCu?kC zo5Tjrl_x@2$SK6Wct}&+U>uCB8L)iZ`E}pAOs6=wbfoVVS37ombp-SOQ(Uz#7lm^k z$FjB%VA5lG<9VKFDFZZEUGC=ppSL>N+gqNi9mh?J4a8iv&y_y|L##N1*46G0e`VMp&4Ki^wBu4g1xr5 zYA7}wFx@;lNZmIYfUh}V&Cp1tXP|wm!_{7ks)E?iYgQfoKq$JGz)Cd-thmBD3BYoW zV6?i=df%dBsm%}@1X#5FGO=O#s&fNN(R9II0;(R_d2MmEVMon(HjJ6y*jHh0G!3i_ z*x&$(%VO5JbA7Qv=`h_qN}XeQ)~D*5U}6IVflPV}%GOx;nqmJ}7E(o{m{et;d=)Oo z6e6&a>2xp{%2f0tN{Gx&l*4Pz&r=S|Lqb)MNg zUdXxn#-gdrT;|6pekFVqa;|(?n%ZCf!=|;Jw0dSb@-pp zl{YAV8g1oq=jvdYPO;u8gcTh>lmyxPZ`!_Y3x(TZ52BMpk0)>PPdD!a9sJ)8rwJ)* z)EalL2Nnk3nb8~OKY&tqk9eRFc`OwR_TR{gE-tRD75x{-YjJg&^Tp|sUvtz`)<)Xe zT$UA1Hxu|vBLc8U9Q~KKtWarQ}zb78lA#VvS9V`Y;Aa*t78^NRn_;yhq&A7 zUXmyh67-ISp;j~sdH@QKrD4(D(AI9^ntWBK=(FCU-{PvKeX;8V&}GbR&Hf6-eDyg8 zXcbvnj-=t8N5a*{VA&s=C`u1ND{ytfii7nc;@+>1@|a{GlV5U~zv@-w$PKMLe+MIZ z6He;j%06jg{(b;qv8L#R<*yo`#og8sz@^h|&AQ;Bn5hpS<*kT?5!n?W<>Kk97~KSW|S|LZ`9q-ML$*+nOIC(%VJ1Xt4Ya`J6mhKDBUW$1AkZ$NC(W{Il?6 zCs?bmDKcXF4juqH8VL~dgLn}k11nvA$Z2kU3|eYSgEo_ohtNMcy=d2!Vxe2UDU0LV z;%kcN-PRbwM7w2T2#Sm-RyJR+sG8=U$;yBwWOYrayW(~qHy(A$he90PXW=5@wrXcv z@8B}F#&$W7Tb_&51SZmT+5 zv&?ASIZ*73id6$EDNaZTPqXn()MT&E5>&jU9`9| zRqi=bC6gb+zakPTx_JC4n~01=#ziPBfDzsROJ?{hiT>$mbj=`G)97+QVwvQlE^BPU zWG4UhZe^1uKMR(VdHdNiGrrBPP814@Q_pdeUBXJhfFxKckzYxHwJLgOcn<&3z=Xx!i`Li#VMyF4CH}{FD{`)sNhN># z)tDzI4&{U@CbP!nz)Il4%F4o;UDnoyfb-{nx}mS1^tuDPEv|@Pi6WC9fzeY;b6P2r z3c@{H^jSHcU3p@|6^pR)P>0;fFTsJxDHYiCI0v z3*7@?EVRDBI2;Rb)1VcubY(X0H|D$<1}mhmm7y+K(~HpuP#X6tpB3Xcmct?ji)Zo& zzZ`+$F>pm?;nx&!n+#d2c$Uxho1c@l+A4py3oNYBy~>*IF4u*T>V&`bVK1*4RX| z#I5~|`yKys#E?E(A}E^-27Lv~R=-H^gUcvbUR76*jII(7w{xBKMkpLsTVJyU2qSMp zkM13PN&c6EMsxQG8m{cop~>l+wY3N58H=lC+wnFqzkZ!`Sz%e-V5tU23-eOiXl*gR z<3??+4qIP+v%QvZS}$UH<6#&&3(MJ=w9z4*JRVGPiyE)!)QE>2RcnXNOMsPMZHHzdpF_$rfCC z8jw@o=7VJmL76r$Xf2{^OB?bzt6g;4?Fbqx@pDi!+W?lhn6pUmE_7^~-MQOqUB}se zJBqN!;~$R&a&vW$@y54d6I^g1hLQH!h3V7oV}G+<$W4Y=mQ$ZrQ&X_dfd zO`*5E4gPu;9J{RV`7=BY_Z{E)&B3<)hmHug{L3}c_t18L<#jjSGSk>)-ABjyJ7sOi zXtTcM&x_dc{R-G{uJIX@Y?A_G5)%X}sc6tz*Brj(T$Z>xm^YKo z6t}K>Q$X;)?Oi=<<5(1p6O&J8;!T`1=8__X5xZM2-da1LV!~qV*0@k1NMXqD2tkHn z;vyJkBpA&&2IImqcB){un`H*%DzlCMfI8Rvt|Z%XCYwmIviRekDo#S09NxR{+;i`F z32JqL?}Z^{4Qi^RIlTE9);37LnhE^@eavS+Gx9Q3wu}~qc^-dJEH!ZQ?pI>oog6!?H zgRPa#BYZ2qc5~~~r&}Ddzq-1;Vqaz-c=s+1TD{_4LUAf)^#OQFF_~R%MIw=deaqDH-md4b*uheIiH@{fvc@xb zzB`;I6BKhQlLN%*A2{RH{Awb;&ZzbF4wX{zh1!NXWO6wn1>LIiaVi&0jd(Q|<^0N4 zWFK|;yLT@p!82lM)<8shI3C?2UM&R@F#vWe>M!N@1B}nnip~<#E(f(KJXw_HHBVSw zW6TP*!jj#oys4U1b^m64vjf40EEGyznzuV~-BRE6&}@IzmOJj*CKf=mgBb#W`_T&U z#H%^OLBcZ1i`|f@)q$2g8XU7utbr;3bntUQ>zl!BGc-Opm!;OLpEz8c9ZG&&6^7B7 z^>z~!i%A3EWL&rvY`2lAQi{pk$`9K|djQ$WlznKq+U{2H;;W9Air~iN0lo{{O)(pU zSja0%F`46H>j1lnVr!}!@!A?DEDvL(TO~0Kl4ACue0*HVvUftxw>is{C~ac?pNNNe zYz2NJ>enI^RYmI;$DY(HfzK<|VZ$&crLF5( zP@!DaA*UGr8-zs~^IDW+3vGev>mXg85~zyeI1yZ#11*pQ;JG0E9OBhNRq{J_wN>>` zENx_QajmFnVJDgkB90mb8uMB#Y-8p*SMsNA>Em=>qgWWvYE&SbXMfEAWo6Dke)azG@q{II}Iq?4Q9=BjKmf9JTxcy(nd9&-h)d}$gd<&+{!;CK(t1iuxdG^ zmXRLMlU4Pm&&s`)f*A2?NyM$ar*zYfI7&G&%V>`&x&~m|jxo#odoV-kILlOSt#SXU zo3>rPg&J06Tz2$(avRFN=Ah|4xO4&p>Za-TW7Sp(Mp8o`1D4!&=dQoZCdMmDwOqP# zYX|cBBacLR)C$y899pbQN?4M=_aLqPSUyA`ZzVT1CtdPY;CjQQ>>R@cvveOygQ@bg zo?$7~2kaZezfiX>vzU2_l^-C#4;tI^$bfYYekPXhEWA$<5p)B0WHDB z%{}{BD+sZ0b06_)$uTr;$&QXy7;RX%NHS)LZjDRzZoi_nAB0US;MLYmTaNI~o^*xF zWbzBom1Nl`u^6wYjYF`o9e0ddws~QorrB#hcFT#EaO&Qgo1gZ z<$(M0mqn}#5ZHBCXGw;AS*O@Alui^yGYn%uaywy)N!sl1SA=PwiKv_kO3$3IGmy?e2!UJZff( Date: Mon, 13 Jun 2022 17:40:31 -0400 Subject: [PATCH 15/24] Result of in is bool, no need to convert --- scripts/v.dissolve/v.dissolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index e32b61090f4..185b809574a 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -461,7 +461,7 @@ def main(): if coltype["type"] not in ("INTEGER", "SMALLINT", "CHARACTER", "TEXT"): gs.fatal(_("Key column must be of type integer or string")) - column_is_str = bool(coltype["type"] in ("CHARACTER", "TEXT")) + column_is_str = coltype["type"] in ("CHARACTER", "TEXT") if columns_to_aggregate and not column_is_str: gs.fatal( _( From c7f26bdff1f0e5318be67620f8a08f2154324cef Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 14 Jun 2022 11:32:33 -0400 Subject: [PATCH 16/24] Remove duplicate columns and methods for non-explicit automatic (interactive) result column handling --- .../tests/v_dissolve_aggregate_test.py | 31 +++++++++++++++++++ scripts/v.dissolve/v.dissolve.py | 15 ++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py index 215776ee401..b5573638983 100644 --- a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py @@ -244,6 +244,37 @@ def test_sqlite_agg_accepted(dataset): assert sorted(aggregate_n) == [1, 2, 3] +def test_duplicate_columns_and_methods_accepted(dataset): + """Duplicate aggregate columns and methods are accepted and deduplicated""" + dissolved_vector = "test_duplicates" + stats = ["count", "count", "n", "min", "min", "n", "sum"] + expected_stats_columns = [ + f"{dataset.float_column_name}_{method}" + for method in ["count", "n", "min", "sum"] + ] + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=[dataset.float_column_name, dataset.float_column_name], + aggregate_method=stats, + aggregate_backend="sql", + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + + columns = gs.vector_columns(dissolved_vector) + assert sorted(columns.keys()) == sorted( + ["cat", dataset.str_column_name] + expected_stats_columns + ), "Unexpected autogenerated column names" + + def test_int_fails(dataset): """An integer column fails with aggregates""" dissolved_vector = "test_int" diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 185b809574a..49ca7091051 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -208,11 +208,24 @@ def check_aggregate_methods_or_fatal(methods, backend): def match_columns_and_methods(columns, methods): - """Return all combinations of columns and methods""" + """Return all combinations of columns and methods + + If a column or a method is specified more than once, only the first occurrence + is used. This makes it suitable for interactive use which values convenience + over predictability. + """ new_columns = [] new_methods = [] + used_columns = [] for column in columns: + if column in used_columns: + continue + used_columns.append(column) + used_methods = [] for method in methods: + if method in used_methods: + continue + used_methods.append(method) new_columns.append(column) new_methods.append(method) return new_columns, new_methods From 98a847da2c9903a14c4160437dc13c44f609a72c Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 15 Jun 2022 09:13:37 -0400 Subject: [PATCH 17/24] Support SQL expressions as columns (as in v.db.update query_column or v.db.select columns) --- .../tests/v_dissolve_aggregate_test.py | 58 ++++++++++ scripts/v.dissolve/v.dissolve.html | 42 ++++--- scripts/v.dissolve/v.dissolve.py | 104 +++++++++++++----- 3 files changed, 162 insertions(+), 42 deletions(-) diff --git a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py index b5573638983..1fbc6310b05 100644 --- a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py @@ -275,6 +275,64 @@ def test_duplicate_columns_and_methods_accepted(dataset): ), "Unexpected autogenerated column names" +def test_sql_expressions_accepted(dataset): + """Arbitrary SQL expressions are accepted for columns""" + dissolved_vector = "test_expressions" + aggregate_columns = ( + f"sum({dataset.float_column_name}), " + f"max({dataset.float_column_name}) - min({dataset.float_column_name}), " + f" count({dataset.float_column_name}) " + ) + result_columns = ( + " sum_of_values double, range_of_values double, count_of_rows integer" + ) + expected_stats_columns = ["sum_of_values", "range_of_values", "count_of_rows"] + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=aggregate_columns, + result_column=result_columns, + aggregate_backend="sql", + ) + + vector_info = gs.vector_info(dissolved_vector) + assert vector_info["level"] == 2 + assert vector_info["centroids"] == 3 + assert vector_info["areas"] == 3 + assert vector_info["num_dblinks"] == 1 + assert vector_info["attribute_primary_key"] == "cat" + + columns = gs.vector_columns(dissolved_vector) + assert sorted(columns.keys()) == sorted( + ["cat", dataset.str_column_name] + expected_stats_columns + ) + + +def test_no_methods_with_univar_and_result_columns_fail(dataset): + """Omitting methods as for sql backend is forbiden for univar""" + dissolved_vector = "test_no_method_univar_fails" + + aggregate_columns = dataset.float_column_name + result_columns = ( + "sum_of_values double,range_of_values double, count_of_rows integer" + ) + assert ( + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=aggregate_columns, + result_column=result_columns, + aggregate_backend="univar", + errors="status", + ) + != 0 + ) + + def test_int_fails(dataset): """An integer column fails with aggregates""" dissolved_vector = "test_int" diff --git a/scripts/v.dissolve/v.dissolve.html b/scripts/v.dissolve/v.dissolve.html index f378f11c063..5afedd58dde 100644 --- a/scripts/v.dissolve/v.dissolve.html +++ b/scripts/v.dissolve/v.dissolve.html @@ -26,11 +26,12 @@

Attribute aggregation

Attributes of merged areas can be aggregated using various aggregation methods -such as sum and mean. The specific methods available depend -on the backend used for aggregation. Two backends are available, -univar and sql. When univar -is used, the methods available are the ones which v.db.univar -uses by default, i.e., n, min, max, range, +such as sum and mean. The specific methods available depend +on the backend used for aggregation. Two aggregate backends (specified in +aggregate_backend) are available, univar and sql. +When univar is used, the methods available are the ones +which v.db.univar uses by default, +i.e., n, min, max, range, mean, mean_abs, variance, stddev, coef_var, and sum. When the sql backend is used, the methods in turn depends on the SQL @@ -46,6 +47,17 @@

Attribute aggregation

The sql aggregate backend, regardless of the underlying database, will typically perform significantly better than the univar backend. +

+Aggregate methods are specified by name in aggregate_method +or using SQL syntax in aggregate_column. +If result_column is provided including type information +and the sql backend is used, +aggregate_column can contain SQL syntax specifying both columns +and the functions applied, e.g., +aggregate_column="sum(cows) / sum(animals)". +In this case, aggregate_methods needs should be omitted. +This provides the highest flexibility and it is suitable for scripting. +

The backend is, by default, determined automatically based on the requested methods. Specifically, the sql backend is used by default, @@ -63,12 +75,6 @@

Attribute aggregation

method (function) name for the backend is recommended because the conversion is a heuristic which may change in the future. -

-Type of the result column is determined based on the method selected. -For n and count, the type is INTEGER and for all other -methods, it is DOUBLE. Aggregate methods which don't produce other types -are not supported unless the value automatically converts to one of these. -

If only aggregate_column is provided, methods default to n, min, max, mean, and sum. @@ -82,13 +88,23 @@

Attribute aggregation

If the result_column is provided, each method is applied only once to the matching column in the aggregate column list and the result will be available under the name of the matching result column. In other words, number -of existing columns, methods, and new columns in aggregate_column, -aggregate_methods, and result_column needs to match and no +of items in aggregate_column, aggregate_method (unless omitted), +and result_column needs to match and no combinations are created on the fly. For scripting, it is recommended to specify all resulting column names, while for interactive use, automatically created combinations are expected to be beneficial, especially for exploratory analysis. +

+Type of the result column is determined based on the method selected. +For n and count, the type is INTEGER and for all other +methods, it is DOUBLE. Aggregate methods which produce other types +require the type to be specified as part of the result_column. +A type can be provided in result_column using the SQL syntax +name type, e.g., sum_of_values double precision. +Type specification is mandatory when SQL syntax is used in +aggregate_column (and aggregate_method is omitted). +

NOTES

GRASS defines a vector area as composite entity consisting of a set of diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 49ca7091051..1192607d1a6 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -62,7 +62,7 @@ # %end # %rules # % requires_all: aggregate_method,aggregate_column -# % requires_all: result_column,aggregate_method,aggregate_column +# % requires_all: result_column,aggregate_column # %end """Dissolve geometries and aggregate attribute values""" @@ -98,7 +98,7 @@ STANDARD_SQL_FUNCTIONS = ["avg", "count", "max", "min", "sum"] -def get_methods_and_backend(methods, backend): +def get_methods_and_backend(methods, backend, provide_defaults): """Get methods and backed based on user-provided methods and backend""" if methods: if not backend: @@ -113,14 +113,15 @@ def get_methods_and_backend(methods, backend): # If all the non-basic functions are available in univar, use it. if in_univar and not neither_in_sql_nor_univar: backend = "univar" - elif backend == "sql": - methods = STANDARD_SQL_FUNCTIONS - elif backend == "univar": - methods = UNIVAR_METHODS - else: - # This is the default SQL functions but using the univar names (and order). - methods = ["n", "min", "max", "mean", "sum"] - backend = "sql" + elif provide_defaults: + if backend == "sql": + methods = STANDARD_SQL_FUNCTIONS + elif backend == "univar": + methods = UNIVAR_METHODS + else: + # This is the default SQL functions but using the univar names (and order). + methods = ["n", "min", "max", "mean", "sum"] + backend = "sql" if not backend: backend = "sql" return methods, backend @@ -196,6 +197,13 @@ def column_value_to_where(column, value, *, quote): def check_aggregate_methods_or_fatal(methods, backend): """Check for known methods if possible or fail""" if backend == "univar": + if not methods: + gs.fatal( + _( + "At least one method must be provided when backend " + "<{backend}> is used" + ).format(backend=backend) + ) for method in methods: if method not in UNIVAR_METHODS: gs.fatal( @@ -232,11 +240,11 @@ def match_columns_and_methods(columns, methods): def create_or_check_result_columns_or_fatal( - result_columns, columns_to_aggregate, methods + result_columns, columns_to_aggregate, methods, backend ): """Create result columns from input if not provided or check them""" if result_columns: - if len(columns_to_aggregate) != len(methods): + if methods and len(columns_to_aggregate) != len(methods): gs.fatal( _( "When result columns are specified, the number of " @@ -258,7 +266,7 @@ def create_or_check_result_columns_or_fatal( columns_to_aggregate=len(columns_to_aggregate), ) ) - if len(result_columns) != len(methods): + if methods and len(result_columns) != len(methods): gs.fatal( _( "The number of result columns ({result_columns}) needs to be " @@ -268,9 +276,27 @@ def create_or_check_result_columns_or_fatal( methods=len(methods), ) ) + if not methods: + if backend == "sql": + for column in result_columns: + if " " not in column: + gs.fatal( + _( + "Type of the result column '{column}' needs a type " + "specified (using the syntax: 'name type') " + "when no methods are provided" + ).format(column=column) + ) + else: + gs.fatal( + _( + "Methods must be specified with {backend} backend " + "and with result columns provided" + ).format(backend=backend) + ) return result_columns return [ - f"{aggregate_column}_{method}" + f"{gs.legalize_vector_name(aggregate_column)}_{method}" for aggregate_column, method in zip(columns_to_aggregate, methods) ] @@ -285,18 +311,30 @@ def aggregate_attributes_sql( result_columns, ): """Aggregate values in selected columns grouped by column using SQL backend""" - if len(columns_to_aggregate) != len(methods) != len(result_columns): + if len(columns_to_aggregate) != len(result_columns): raise ValueError( - "Number of columns_to_aggregate, methods, and result_columns " - "must be the same" + "Number of columns_to_aggregate and result_columns must be the same" ) - select_columns = [ - f"{method}({agg_column})" - for method, agg_column in zip(methods, columns_to_aggregate) - ] - column_types = [ - "INTEGER" if method == "count" else "DOUBLE" for method in methods - ] * len(columns_to_aggregate) + if methods and len(columns_to_aggregate) != len(methods): + raise ValueError("Number of columns_to_aggregate and methods must be the same") + if not methods: + for result_column in result_columns: + if " " not in result_column: + raise ValueError( + f"Column {result_column} from result_columns without type" + ) + if methods: + select_columns = [ + f"{method}({agg_column})" + for method, agg_column in zip(methods, columns_to_aggregate) + ] + column_types = [ + "INTEGER" if method == "count" else "DOUBLE" for method in methods + ] * len(columns_to_aggregate) + else: + select_columns = columns_to_aggregate + column_types = None + records = json.loads( gs.read_command( "v.db.select", @@ -309,14 +347,20 @@ def aggregate_attributes_sql( )["records"] updates = [] add_columns = [] - for result_column, column_type in zip(result_columns, column_types): - add_columns.append(f"{result_column} {column_type}") + if column_types: + for result_column, column_type in zip(result_columns, column_types): + add_columns.append(f"{result_column} {column_type}") + else: + add_columns = result_columns.copy() for row in records: where = column_value_to_where(column, row[column], quote=quote_column) for ( result_column, key, ) in zip(result_columns, select_columns): + if not column_types: + # Column types are part of the result column name list. + result_column = result_column.split(" ", maxsplit=1)[0] updates.append( { "column": result_column, @@ -405,7 +449,7 @@ def option_as_list(options, name): """Get value of an option as a list""" option = options[name] if option: - return option.split(",") + return [value.strip() for value in option.split(",")] return [] @@ -420,10 +464,11 @@ def main(): columns_to_aggregate = option_as_list(options, "aggregate_column") user_aggregate_methods = option_as_list(options, "aggregate_method") + result_columns = option_as_list(options, "result_column") + user_aggregate_methods, aggregate_backend = get_methods_and_backend( - user_aggregate_methods, aggregate_backend + user_aggregate_methods, aggregate_backend, provide_defaults=not result_columns ) - result_columns = option_as_list(options, "result_column") if not result_columns: columns_to_aggregate, user_aggregate_methods = match_columns_and_methods( columns_to_aggregate, user_aggregate_methods @@ -436,6 +481,7 @@ def main(): result_columns=result_columns, columns_to_aggregate=columns_to_aggregate, methods=user_aggregate_methods, + backend=aggregate_backend, ) # does map exist? From 18786dd83e86895ce3dfe08582142b6e62adeb64 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Mon, 20 Jun 2022 15:21:12 -0400 Subject: [PATCH 18/24] Support also text-returning aggregate function such as SQLite group_concat. --- .../tests/v_dissolve_aggregate_test.py | 29 ++++++++- scripts/v.dissolve/v.dissolve.py | 62 +++++++++++++++++-- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py index 1fbc6310b05..2d273ce5b27 100644 --- a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py @@ -179,7 +179,7 @@ def test_aggregate_column_result(dataset, backend): def test_sqlite_agg_accepted(dataset): - """Numeric SQLite aggregate function are accepted + """Numeric SQLite aggregate functions are accepted Additionally, it checks: 1. generated column names @@ -244,6 +244,33 @@ def test_sqlite_agg_accepted(dataset): assert sorted(aggregate_n) == [1, 2, 3] +def test_sqlite_concat(dataset): + """SQLite concat text-returning aggregate function works""" + dissolved_vector = "test_sqlite_concat" + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=f"group_concat({dataset.int_column_name})", + result_column="concat_values text", + aggregate_backend="sql", + ) + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + format="json", + ) + )["records"] + # Order of records is ignored - they are just sorted. + # Order within values of group_concat is defined as arbitrary by SQLite. + expected_integers = sorted(["10", "10,10,24", "5,5"]) + actual_integers = sorted([record["concat_values"] for record in records]) + for expected, actual in zip(expected_integers, actual_integers): + assert sorted(expected.split(",")) == sorted(actual.split(",")) + + def test_duplicate_columns_and_methods_accepted(dataset): """Duplicate aggregate columns and methods are accepted and deduplicated""" dissolved_vector = "test_duplicates" diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 1192607d1a6..e8e65089cbe 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -153,12 +153,33 @@ def modify_methods_for_backend(methods, backend): return new_methods +def quote_from_type(column_type): + """Returns quote if column values need to be quoted based on their type + + Defaults to quoting for unknown types and no quoting for falsely values, + i.e., unknown types are assumed to be in need of quoting while missing type + information is assumed to be associated with numbers which don't need quoting. + """ + # Needs a general solution, e.g., https://github.com/OSGeo/grass/pull/1110 + if not column_type or column_type.upper() in [ + "INT", + "INTEGER", + "SMALLINT", + "REAL", + "DOUBLE", + "DOUBLE PRECISION", + ]: + return "" + return "'" + + def updates_to_sql(table, updates): """Create SQL from a list of dicts with column, value, where""" sql = ["BEGIN TRANSACTION"] for update in updates: + quote = quote_from_type(update.get("type", None)) sql.append( - f"UPDATE {table} SET {update['column']} = {update['value']} " + f"UPDATE {table} SET {update['column']} = {quote}{update['value']}{quote} " f"WHERE {update['where']};" ) sql.append("END TRANSACTION") @@ -215,6 +236,27 @@ def check_aggregate_methods_or_fatal(methods, backend): # and open for SQLite depending on its extensions. +def aggregate_columns_exist_or_fatal(vector, layer, columns): + """Check that all columns exist or end with fatal error""" + column_names = gs.vector_columns(vector, layer).keys() + for column in columns: + if column not in column_names: + if "(" in column: + gs.fatal( + _( + "Column <{column}> does not exist in vector <{vector}> " + "(layer <{layer}>). Specify result columns if you are adding " + "function calls to aggregate columns." + ).format(vector=vector, layer=layer, column=column) + ) + gs.fatal( + _( + "Column <{column}> selected for aggregation does not exist " + "in vector <{vector}> (layer <{layer}>)" + ).format(vector=vector, layer=layer, column=column) + ) + + def match_columns_and_methods(columns, methods): """Return all combinations of columns and methods @@ -351,19 +393,26 @@ def aggregate_attributes_sql( for result_column, column_type in zip(result_columns, column_types): add_columns.append(f"{result_column} {column_type}") else: - add_columns = result_columns.copy() + # Column types are part of the result column name list. + add_columns = result_columns.copy() # Ensure we have our own copy. + # Split column definitions into two lists. + result_columns = [] + column_types = [] + for definition in add_columns: + column_name, column_type = definition.split(" ", maxsplit=1) + result_columns.append(column_name) + column_types.append(column_type) for row in records: where = column_value_to_where(column, row[column], quote=quote_column) for ( result_column, + column_type, key, - ) in zip(result_columns, select_columns): - if not column_types: - # Column types are part of the result column name list. - result_column = result_column.split(" ", maxsplit=1)[0] + ) in zip(result_columns, column_types, select_columns): updates.append( { "column": result_column, + "type": column_type, "value": row[key], "where": where, } @@ -470,6 +519,7 @@ def main(): user_aggregate_methods, aggregate_backend, provide_defaults=not result_columns ) if not result_columns: + aggregate_columns_exist_or_fatal(input_vector, layer, columns_to_aggregate) columns_to_aggregate, user_aggregate_methods = match_columns_and_methods( columns_to_aggregate, user_aggregate_methods ) From fbc0740c77e994d9f4696c1e14f16520cced26e1 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Mon, 17 Jul 2023 16:23:46 -0400 Subject: [PATCH 19/24] Add examples to documentation, use plural for columns and methods --- scripts/v.dissolve/v.dissolve.html | 118 ++++++++++++++++++++++++++--- scripts/v.dissolve/v.dissolve.py | 50 ++++++++---- 2 files changed, 142 insertions(+), 26 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.html b/scripts/v.dissolve/v.dissolve.html index 5afedd58dde..90f950dce9a 100644 --- a/scripts/v.dissolve/v.dissolve.html +++ b/scripts/v.dissolve/v.dissolve.html @@ -48,13 +48,13 @@

Attribute aggregation

will typically perform significantly better than the univar backend.

-Aggregate methods are specified by name in aggregate_method -or using SQL syntax in aggregate_column. -If result_column is provided including type information +Aggregate methods are specified by name in aggregate_methods +or using SQL syntax in aggregate_columns. +If result_columns is provided including type information and the sql backend is used, -aggregate_column can contain SQL syntax specifying both columns +aggregate_columns can contain SQL syntax specifying both columns and the functions applied, e.g., -aggregate_column="sum(cows) / sum(animals)". +aggregate_columns="sum(cows) / sum(animals)". In this case, aggregate_methods needs should be omitted. This provides the highest flexibility and it is suitable for scripting. @@ -76,19 +76,19 @@

Attribute aggregation

is a heuristic which may change in the future.

-If only aggregate_column is provided, methods default to +If only aggregate_columns is provided, methods default to n, min, max, mean, and sum. If the univar backend is specified, all the available methods for the univar backend are used.

-If the result_column is not provided, each method is applied to each +If the result_columns is not provided, each method is applied to each specified column producing result columns for all combinations. These result columns have auto-generated names based on the aggregate column and method. If the result_column is provided, each method is applied only once to the matching column in the aggregate column list and the result will be available under the name of the matching result column. In other words, number -of items in aggregate_column, aggregate_method (unless omitted), +of items in aggregate_columns, aggregate_methods (unless omitted), and result_column needs to match and no combinations are created on the fly. For scripting, it is recommended to specify all resulting column names, @@ -99,11 +99,11 @@

Attribute aggregation

Type of the result column is determined based on the method selected. For n and count, the type is INTEGER and for all other methods, it is DOUBLE. Aggregate methods which produce other types -require the type to be specified as part of the result_column. -A type can be provided in result_column using the SQL syntax +require the type to be specified as part of the result_columns. +A type can be provided in result_columns using the SQL syntax name type, e.g., sum_of_values double precision. Type specification is mandatory when SQL syntax is used in -aggregate_column (and aggregate_method is omitted). +aggregate_columns (and aggregate_methods is omitted).

NOTES

@@ -155,6 +155,102 @@

Dissolving adjacent SHAPE files to remove tile boundaries

v.dissolve input=clc2000_clean output=clc2000_final col=CODE_00 +

Attribute aggregation

+ +While dissolving, we can aggregate attribute values of the original features. +Let's aggregate area in acres (ACRES) of all municipal boundaries +(boundary_municp) in the full NC dataset while dissolving common boundaries +based on the name in the DOTURBAN_N column +(long lines are split with backslash marking continued line as in Bash): + +
+v.dissolve input=boundary_municp column=DOTURBAN_N output=municipalities \
+    aggregate_columns=ACRES
+
+ +The above will create multiple columns for each of the statistics computed +by default. We can limit the number of statistics computed by specifying +the method which should be used: + +
+v.dissolve input=boundary_municp column=DOTURBAN_N output=municipalities_2 \
+    aggregate_columns=ACRES aggregate_methods=sum
+
+ +The above gives a single column with the sum for all values in the ACRES column +for each group of original features which had the same value in the DOTURBAN_N +column and are now dissolved (merged) into one. + +

Aggregating multiple attributes

+ +Expanding on the previous example, we can compute values for multiple columns +at once by adding more columns to the aggregate_columns option. +We will compute average of values in the NEW_PERC_G column: + +
+v.dissolve input=boundary_municp column=DOTURBAN_N output=municipalities_3 \
+    aggregate_columns=ACRES,NEW_PERC_G aggregate_methods=sum,avg
+
+ +By default, all methods specified in the aggregate_methods are applied +to all columns, so result of the above is four columns. +While this is convenient for getting multiple statistics for similar columns +(e.g. averages and standard deviations of multiple population statistics columns), +in our case, each column is different and each aggregate method should be +applied only to its corresponding column. + +

+The v.dissolve module will apply each aggregate method only to the +corresponding column when column names for the results are specified manually +with the result_columns option: + +

+v.dissolve input=boundary_municp column=DOTURBAN_N output=municipalities_4 \
+    aggregate_columns=ACRES,NEW_PERC_G aggregate_methods=sum,avg \
+    result_columns=acres,new_perc_g
+
+ +Now we have full control over what columns are created, but we also need to specify +an aggregate method for each column even when the aggregate methods are the same: + +
+v.dissolve input=boundary_municp column=DOTURBAN_N output=municipalities_5 \
+    aggregate_columns=ACRES,DOTURBAN_N,TEXT_NAME aggregate_methods=sum,count,count \
+    result_columns=acres,number_of_parts,named_parts
+
+ +While it is often not necessary to specify aggregate methods or names for +interactive exploratory analysis, specifying both aggregate_methods +and result_columns manually is a best practice for scripting. + +

Aggregating using SQL syntax

+ +The aggregation can be done also using the full SQL syntax and set of aggregate +functions available for a given attribute database backend. +Here, we will assume the default SQLite database backend for attribute. + +

+Modifying the previous example, we will now specify the SQL aggregate function calls +explicitly instead of letting v.dissolve generate them for us. +We will compute sum of the ACRES column using sum(ACRES) +(alternatively, we could use SQLite specific total(ACRES) +which returns zero even when all values are NULL). +Further, we will count number of aggregated (i.e., dissolved) parts using +count(*) which counts all rows regardless of NULL values. +Then, we will count all unique names of parts as distinguished by +the MB_NAME column using count(distinct MB_NAME). +Finally, we will collect all these names into a comma-separated list using +group_concat(MB_NAME): + +

+v.dissolve input=boundary_municp column=DOTURBAN_N output=municipalities_6 \
+    aggregate_columns="total(ACRES),count(*),count(distinct MB_NAME),group_concat(MB_NAME)" \
+    result_columns="acres REAL,named_parts INTEGER,unique_names INTEGER,names TEXT"
+
+ +Here, v.dissolve doesn't make any assumptions about the resulting +column types, so we specified both named and the type of each column. +

SEE ALSO

diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index e8e65089cbe..c41ecc6ff3d 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -9,7 +9,7 @@ # Vaclav Petras (aggregate statistics) # PURPOSE: Dissolve common boundaries between areas with common cat # (frontend to v.extract -d) -# COPYRIGHT: (c) 2006-2022 Hamish Bowman, and the GRASS Development Team +# COPYRIGHT: (c) 2006-2023 Hamish Bowman, and the GRASS Development Team # This program is free software under the GNU General Public # License (>=v2). Read the file COPYING that comes with GRASS # for details. @@ -24,32 +24,39 @@ # % keyword: line # %end # %option G_OPT_V_INPUT +# % guisection: Dissolving # %end # %option G_OPT_V_FIELD # % label: Layer number or name. # % required: no +# % guisection: Dissolving # %end # %option G_OPT_DB_COLUMN # % description: Name of attribute column used to dissolve common boundaries +# % guisection: Dissolving # %end # %option G_OPT_V_OUTPUT +# % guisection: Dissolving # %end # %option G_OPT_DB_COLUMN -# % key: aggregate_column +# % key: aggregate_columns # % label: Name of attribute columns to get aggregate statistics for # % description: One column per method if result columns are specified +# % guisection: Aggregation # % multiple: yes # %end # %option -# % key: aggregate_method +# % key: aggregate_methods # % label: Aggregate statistics method (e.g., sum) # % description: Default is all available basic statistics for a given backend +# % guisection: Aggregation # % multiple: yes # %end # %option G_OPT_DB_COLUMN -# % key: result_column +# % key: result_columns # % label: New attribute column name for aggregate statistics results # % description: Defaults to aggregate column name and statistics name +# % guisection: Aggregation # % multiple: yes # %end # %option @@ -59,10 +66,11 @@ # % multiple: no # % required: no # % options: sql,univar +# % guisection: Aggregation # %end # %rules -# % requires_all: aggregate_method,aggregate_column -# % requires_all: result_column,aggregate_column +# % requires_all: aggregate_methods,aggregate_columns +# % requires_all: result_columns,aggregate_columns # %end """Dissolve geometries and aggregate attribute values""" @@ -245,8 +253,9 @@ def aggregate_columns_exist_or_fatal(vector, layer, columns): gs.fatal( _( "Column <{column}> does not exist in vector <{vector}> " - "(layer <{layer}>). Specify result columns if you are adding " - "function calls to aggregate columns." + "(layer <{layer}>). Specify result columns with 'name type' " + "syntax if you are using function calls instead of aggregate " + "column names only." ).format(vector=vector, layer=layer, column=column) ) gs.fatal( @@ -324,10 +333,15 @@ def create_or_check_result_columns_or_fatal( if " " not in column: gs.fatal( _( - "Type of the result column '{column}' needs a type " + "Result column '{column}' needs a type " "specified (using the syntax: 'name type') " - "when no methods are provided" - ).format(column=column) + "when no methods are provided with the " + "{option_name} option and aggregation backend is '{backend}'" + ).format( + column=column, + option_name="aggregate_methods", + backend=backend, + ) ) else: gs.fatal( @@ -491,9 +505,15 @@ def cleanup(name): name=name, quiet=True, stderr=subprocess.DEVNULL, + errors="ignore", ) +def remove_mapset_from_name(name): + """Remove the at-mapset part (if any) from the name""" + return name.split("@", maxsplit=1)[0] + + def option_as_list(options, name): """Get value of an option as a list""" option = options[name] @@ -511,9 +531,9 @@ def main(): column = options["column"] aggregate_backend = options["aggregate_backend"] - columns_to_aggregate = option_as_list(options, "aggregate_column") - user_aggregate_methods = option_as_list(options, "aggregate_method") - result_columns = option_as_list(options, "result_column") + columns_to_aggregate = option_as_list(options, "aggregate_columns") + user_aggregate_methods = option_as_list(options, "aggregate_methods") + result_columns = option_as_list(options, "result_columns") user_aggregate_methods, aggregate_backend = get_methods_and_backend( user_aggregate_methods, aggregate_backend, provide_defaults=not result_columns @@ -579,7 +599,7 @@ def main(): ).format(column_type=coltype["type"]) ) - tmpfile = gs.append_node_pid(output) + tmpfile = gs.append_node_pid(remove_mapset_from_name(output)) atexit.register(cleanup, tmpfile) try: From c8801759e13c69c49da6128b2968fd80ee38f240 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Mon, 17 Jul 2023 16:24:40 -0400 Subject: [PATCH 20/24] Add simple SQL escape function to double single quotes --- scripts/v.dissolve/v.dissolve.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index c41ecc6ff3d..c3edd1e3036 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -181,13 +181,29 @@ def quote_from_type(column_type): return "'" +def sql_escape(text): + """Escape string for use in SQL statement. + + If the argument is not string, it is returned as is. + + Simple support for direct creation of SQL statements. This function, + column_value_to_where, and updates_to_sql need a rewrite with a more systematic + solution for generating statements in Python for GRASS GIS attribute engine. + """ + if isinstance(text, str): + return text.replace("'", "''") + return text + + def updates_to_sql(table, updates): """Create SQL from a list of dicts with column, value, where""" sql = ["BEGIN TRANSACTION"] for update in updates: quote = quote_from_type(update.get("type", None)) + value = update["value"] + sql_value = f"{quote}{sql_escape(value) if value else 'NULL'}{quote}" sql.append( - f"UPDATE {table} SET {update['column']} = {quote}{update['value']}{quote} " + f"UPDATE {table} SET {update['column']} = {sql_value} " f"WHERE {update['where']};" ) sql.append("END TRANSACTION") @@ -219,7 +235,7 @@ def column_value_to_where(column, value, *, quote): if value is None: return f"{column} IS NULL" if quote: - return f"{column}='{value}'" + return f"{column}='{sql_escape(value)}'" return f"{column}={value}" From 1d5dfee6a66b47dfb51b1b92e1f5ecb5a5c9307e Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Mon, 17 Jul 2023 16:33:38 -0400 Subject: [PATCH 21/24] Flip some conditions for better readability and less indentation --- scripts/v.dissolve/v.dissolve.py | 119 ++++++++++++++++--------------- 1 file changed, 60 insertions(+), 59 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index c3edd1e3036..ad5f165a4eb 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -310,67 +310,68 @@ def create_or_check_result_columns_or_fatal( result_columns, columns_to_aggregate, methods, backend ): """Create result columns from input if not provided or check them""" - if result_columns: - if methods and len(columns_to_aggregate) != len(methods): - gs.fatal( - _( - "When result columns are specified, the number of " - "aggregate columns ({columns_to_aggregate}) needs to be " - "the same as the number of methods ({methods})" - ).format( - columns_to_aggregate=len(columns_to_aggregate), - methods=len(methods), - ) + if not result_columns: + return [ + f"{gs.legalize_vector_name(aggregate_column)}_{method}" + for aggregate_column, method in zip(columns_to_aggregate, methods) + ] + + if methods and len(columns_to_aggregate) != len(methods): + gs.fatal( + _( + "When result columns are specified, the number of " + "aggregate columns ({columns_to_aggregate}) needs to be " + "the same as the number of methods ({methods})" + ).format( + columns_to_aggregate=len(columns_to_aggregate), + methods=len(methods), ) - if len(result_columns) != len(columns_to_aggregate): - gs.fatal( - _( - "The number of result columns ({result_columns}) needs to be " - "the same as the number of aggregate columns " - "({columns_to_aggregate})" - ).format( - result_columns=len(result_columns), - columns_to_aggregate=len(columns_to_aggregate), - ) + ) + if len(result_columns) != len(columns_to_aggregate): + gs.fatal( + _( + "The number of result columns ({result_columns}) needs to be " + "the same as the number of aggregate columns " + "({columns_to_aggregate})" + ).format( + result_columns=len(result_columns), + columns_to_aggregate=len(columns_to_aggregate), + ) + ) + if methods and len(result_columns) != len(methods): + gs.fatal( + _( + "The number of result columns ({result_columns}) needs to be " + "the same as the number of aggregation methods ({methods})" + ).format( + result_columns=len(result_columns), + methods=len(methods), ) - if methods and len(result_columns) != len(methods): + ) + if not methods: + if backend == "sql": + for column in result_columns: + if " " not in column: + gs.fatal( + _( + "Result column '{column}' needs a type " + "specified (using the syntax: 'name type') " + "when no methods are provided with the " + "{option_name} option and aggregation backend is '{backend}'" + ).format( + column=column, + option_name="aggregate_methods", + backend=backend, + ) + ) + else: gs.fatal( _( - "The number of result columns ({result_columns}) needs to be " - "the same as the number of aggregation methods ({methods})" - ).format( - result_columns=len(result_columns), - methods=len(methods), - ) + "Methods must be specified with {backend} backend " + "and with result columns provided" + ).format(backend=backend) ) - if not methods: - if backend == "sql": - for column in result_columns: - if " " not in column: - gs.fatal( - _( - "Result column '{column}' needs a type " - "specified (using the syntax: 'name type') " - "when no methods are provided with the " - "{option_name} option and aggregation backend is '{backend}'" - ).format( - column=column, - option_name="aggregate_methods", - backend=backend, - ) - ) - else: - gs.fatal( - _( - "Methods must be specified with {backend} backend " - "and with result columns provided" - ).format(backend=backend) - ) - return result_columns - return [ - f"{gs.legalize_vector_name(aggregate_column)}_{method}" - for aggregate_column, method in zip(columns_to_aggregate, methods) - ] + return result_columns def aggregate_attributes_sql( @@ -533,9 +534,9 @@ def remove_mapset_from_name(name): def option_as_list(options, name): """Get value of an option as a list""" option = options[name] - if option: - return [value.strip() for value in option.split(",")] - return [] + if not option: + return [] + return [value.strip() for value in option.split(",")] def main(): From 10818e2fcde2cd9a4738517689ac83df34f0b51e Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 18 Jul 2023 09:51:06 -0400 Subject: [PATCH 22/24] Improve author lists --- scripts/v.dissolve/v.dissolve.html | 4 ++-- scripts/v.dissolve/v.dissolve.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.html b/scripts/v.dissolve/v.dissolve.html index 90f950dce9a..64abbe55311 100644 --- a/scripts/v.dissolve/v.dissolve.html +++ b/scripts/v.dissolve/v.dissolve.html @@ -262,7 +262,7 @@

SEE ALSO

AUTHORS

-M. Hamish Bowman, Dept. Marine Science, Otago University, New Zealand (module)
+M. Hamish Bowman, Department of Marine Science, Otago University, New Zealand (module)
Markus Neteler (column support)
Trevor Wiens (help page)
-Vaclav Petras, NC State University, Center for Geospatial Analytics (aggregate statistics) +Vaclav Petras, NC State University, Center for Geospatial Analytics, GeoForAll Lab (aggregate statistics) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index ad5f165a4eb..b767c73ca58 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -2,8 +2,7 @@ ############################################################################ # # MODULE: v.dissolve -# AUTHOR: M. Hamish Bowman, Dept. Marine Science, Otago University, -# New Zealand +# AUTHOR: M. Hamish Bowman, Dept. Marine Science, Otago University # Markus Neteler for column support # Converted to Python by Glynn Clements # Vaclav Petras (aggregate statistics) From be56efdda112fca46fb778168d21a379c5057f7f Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 18 Jul 2023 11:23:22 -0400 Subject: [PATCH 23/24] Support general SQL syntax just like v.db.select for the price of less checks. Now depends on v.db.select producing the list of column names #3090. Test and example included. --- .../tests/v_dissolve_aggregate_test.py | 30 +++++++++++- scripts/v.dissolve/v.dissolve.html | 48 +++++++++++++++++-- scripts/v.dissolve/v.dissolve.py | 19 +++++--- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py index 2d273ce5b27..1c2b6d45123 100644 --- a/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py +++ b/scripts/v.dissolve/tests/v_dissolve_aggregate_test.py @@ -245,7 +245,7 @@ def test_sqlite_agg_accepted(dataset): def test_sqlite_concat(dataset): - """SQLite concat text-returning aggregate function works""" + """SQLite group concat text-returning aggregate function works""" dissolved_vector = "test_sqlite_concat" gs.run_command( "v.dissolve", @@ -271,6 +271,34 @@ def test_sqlite_concat(dataset): assert sorted(expected.split(",")) == sorted(actual.split(",")) +def test_sqlite_concat_with_two_parameters(dataset): + """SQLite group concat text-returning two-parameter aggregate function works""" + dissolved_vector = "test_sqlite_concat_separator" + separator = "--+--" + gs.run_command( + "v.dissolve", + input=dataset.vector_name, + column=dataset.str_column_name, + output=dissolved_vector, + aggregate_column=f"group_concat({dataset.int_column_name}, '{separator}')", + result_column="concat_values text", + aggregate_backend="sql", + ) + records = json.loads( + gs.read_command( + "v.db.select", + map=dissolved_vector, + format="json", + ) + )["records"] + # Order of records is ignored - they are just sorted. + # Order within values of group_concat is defined as arbitrary by SQLite. + expected_integers = sorted(["10", "10,10,24", "5,5"]) + actual_integers = sorted([record["concat_values"] for record in records]) + for expected, actual in zip(expected_integers, actual_integers): + assert sorted(expected.split(",")) == sorted(actual.split(separator)) + + def test_duplicate_columns_and_methods_accepted(dataset): """Duplicate aggregate columns and methods are accepted and deduplicated""" dissolved_vector = "test_duplicates" diff --git a/scripts/v.dissolve/v.dissolve.html b/scripts/v.dissolve/v.dissolve.html index 64abbe55311..a07428bb0fd 100644 --- a/scripts/v.dissolve/v.dissolve.html +++ b/scripts/v.dissolve/v.dissolve.html @@ -55,7 +55,7 @@

Attribute aggregation

aggregate_columns can contain SQL syntax specifying both columns and the functions applied, e.g., aggregate_columns="sum(cows) / sum(animals)". -In this case, aggregate_methods needs should be omitted. +In this case, aggregate_methods should to be omitted. This provides the highest flexibility and it is suitable for scripting.

@@ -168,7 +168,21 @@

Attribute aggregation

aggregate_columns=ACRES -The above will create multiple columns for each of the statistics computed +To inspect the result, we will use v.db.select retrieving only one row +for DOTURBAN_N == 'Wadesboro': + +
+v.db.select municipalities where="DOTURBAN_N == 'Wadesboro'" separator=tab
+
+ +The resulting table may look like this: + +
+cat  DOTURBAN_N    ACRES_n    ACRES_min    ACRES_max    ACRES_mean    ACRES_sum
+66   Wadesboro     2          634.987      3935.325     2285.156      4570.312
+
+ +The above created multiple columns for each of the statistics computed by default. We can limit the number of statistics computed by specifying the method which should be used: @@ -221,7 +235,8 @@

Aggregating multiple attributes

While it is often not necessary to specify aggregate methods or names for interactive exploratory analysis, specifying both aggregate_methods -and result_columns manually is a best practice for scripting. +and result_columns manually is a best practice for scripting +(unless SQL syntax is used for aggregate_columns, see below).

Aggregating using SQL syntax

@@ -251,6 +266,33 @@

Aggregating using SQL syntax

Here, v.dissolve doesn't make any assumptions about the resulting column types, so we specified both named and the type of each column. +

+When working with general SQL syntax, v.dissolve turns off its checks for +number of aggregate and result columns to allow for all SQL syntax to be used +for aggregate columns. This allows us to use also functions with multiple parameters, +for example specify separator to be used with group_concat: + +

+    v.dissolve input=boundary_municp column=DOTURBAN_N output=municipalities_7 \
+        aggregate_columns="group_concat(MB_NAME, ';')" \
+        result_columns="names TEXT"
+
+ +To inspect the result, we will use v.db.select retrieving only one row +for DOTURBAN_N == 'Wadesboro': + +
+v.db.select municipalities_7 where="DOTURBAN_N == 'Wadesboro'" separator=tab
+
+ +The resulting table may look like this: + +
+cat	DOTURBAN_N	names
+66	Wadesboro	Wadesboro;Lilesville
+
+ +

SEE ALSO

diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index b767c73ca58..cd699db380a 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -326,7 +326,12 @@ def create_or_check_result_columns_or_fatal( methods=len(methods), ) ) - if len(result_columns) != len(columns_to_aggregate): + # When methods are not set with sql backend, we might be dealing with the general + # SQL syntax provided for columns, so we can't parse that easily, so let's not + # check that here. + if (methods or backend != "sql") and len(result_columns) != len( + columns_to_aggregate + ): gs.fatal( _( "The number of result columns ({result_columns}) needs to be " @@ -383,7 +388,7 @@ def aggregate_attributes_sql( result_columns, ): """Aggregate values in selected columns grouped by column using SQL backend""" - if len(columns_to_aggregate) != len(result_columns): + if methods and len(columns_to_aggregate) != len(result_columns): raise ValueError( "Number of columns_to_aggregate and result_columns must be the same" ) @@ -407,7 +412,7 @@ def aggregate_attributes_sql( select_columns = columns_to_aggregate column_types = None - records = json.loads( + data = json.loads( gs.read_command( "v.db.select", map=input_name, @@ -416,7 +421,9 @@ def aggregate_attributes_sql( group=column, format="json", ) - )["records"] + ) + # We added the group column to the select, so we need to skip it here. + select_column_names = [item["name"] for item in data["info"]["columns"]][1:] updates = [] add_columns = [] if column_types: @@ -432,13 +439,13 @@ def aggregate_attributes_sql( column_name, column_type = definition.split(" ", maxsplit=1) result_columns.append(column_name) column_types.append(column_type) - for row in records: + for row in data["records"]: where = column_value_to_where(column, row[column], quote=quote_column) for ( result_column, column_type, key, - ) in zip(result_columns, column_types, select_columns): + ) in zip(result_columns, column_types, select_column_names): updates.append( { "column": result_column, From 9dc9867f89dd2c5fd4bf9516e76ed712755acf25 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 19 Jul 2023 09:59:40 -0400 Subject: [PATCH 24/24] Add more description and see also --- scripts/v.dissolve/v.dissolve.html | 4 +++- scripts/v.dissolve/v.dissolve.py | 13 +++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/v.dissolve/v.dissolve.html b/scripts/v.dissolve/v.dissolve.html index a07428bb0fd..e22290fa064 100644 --- a/scripts/v.dissolve/v.dissolve.html +++ b/scripts/v.dissolve/v.dissolve.html @@ -299,7 +299,9 @@

SEE ALSO

v.category, v.centroids, v.extract, -v.reclass +v.reclass, +v.db.univar, +v.db.select

AUTHORS

diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index cd699db380a..641c48c5b66 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -39,32 +39,33 @@ # %end # %option G_OPT_DB_COLUMN # % key: aggregate_columns -# % label: Name of attribute columns to get aggregate statistics for -# % description: One column per method if result columns are specified +# % label: Names of attribute columns to get aggregate statistics for +# % description: One column name or SQL expression per method if result columns are specified # % guisection: Aggregation # % multiple: yes # %end # %option # % key: aggregate_methods # % label: Aggregate statistics method (e.g., sum) -# % description: Default is all available basic statistics for a given backend +# % description: Default is all available basic statistics for a given backend (for sql backend: avg, count, max, min, sum) # % guisection: Aggregation # % multiple: yes # %end # %option G_OPT_DB_COLUMN # % key: result_columns -# % label: New attribute column name for aggregate statistics results -# % description: Defaults to aggregate column name and statistics name +# % label: New attribute column names for aggregate statistics results +# % description: Defaults to aggregate column name and statistics name and can contain type # % guisection: Aggregation # % multiple: yes # %end # %option # % key: aggregate_backend # % label: Backend for attribute aggregation -# % description: Default is sql unless the methods are for univar +# % description: Default is sql unless the provided aggregate methods are for univar # % multiple: no # % required: no # % options: sql,univar +# % descriptions: sql;Uses SQL attribute database;univar;Uses v.db.univar # % guisection: Aggregation # %end # %rules