From a92e7fd618918f158030ad436513baf12baae514 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Fri, 3 May 2024 15:31:55 -0400 Subject: [PATCH 1/4] pythonlib/parse_command: automatically parse json and csv output --- python/grass/script/core.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/python/grass/script/core.py b/python/grass/script/core.py index 64355616954..400c36dc235 100644 --- a/python/grass/script/core.py +++ b/python/grass/script/core.py @@ -27,6 +27,9 @@ import string import random import shlex +import json +import csv +import io from tempfile import NamedTemporaryFile from pathlib import Path @@ -543,26 +546,35 @@ def read_command(*args, **kwargs): def parse_command(*args, **kwargs): """Passes all arguments to read_command, then parses the output - by parse_key_val(). + by default with parse_key_val(). - Parsing function can be optionally given by parse parameter - including its arguments, e.g. + If the command has parameter format and is called with + format=json, the output will be parsed into a dictionary. + Similarly, with format=csv the output will be parsed into + a list of lists (CSV rows). :: - parse_command(..., parse = (grass.parse_key_val, { 'sep' : ':' })) + parse_command("v.db.select", ..., format="json") - or you can simply define delimiter + Custom parsing function can be optionally given by parse parameter + including its arguments, e.g. :: - parse_command(..., delimiter = ':') + parse_command(..., parse=(gs.parse_key_val, {'sep': ':'})) + + Parameter delimiter is deprecated. :param args: list of unnamed arguments (see start_command() for details) :param kwargs: list of named arguments (see start_command() for details) :return: parsed module output """ + + def parse_csv(result): + return list(csv.reader(io.StringIO(result))) + parse = None parse_args = {} if "parse" in kwargs: @@ -570,13 +582,16 @@ def parse_command(*args, **kwargs): parse = kwargs["parse"][0] parse_args = kwargs["parse"][1] del kwargs["parse"] - - if "delimiter" in kwargs: - parse_args = {"sep": kwargs["delimiter"]} - del kwargs["delimiter"] + elif kwargs.get("format") == "json": + parse = json.loads + elif kwargs.get("format") == "csv": + parse = parse_csv if not parse: parse = parse_key_val # use default fn + if "delimiter" in kwargs: + parse_args = {"sep": kwargs["delimiter"]} + del kwargs["delimiter"] res = read_command(*args, **kwargs) From fb0dd531e890577891373c457c099dabeb0a5a07 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Fri, 3 May 2024 16:42:56 -0400 Subject: [PATCH 2/4] use csv.DictReader instead --- python/grass/script/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/grass/script/core.py b/python/grass/script/core.py index 400c36dc235..b41b0e824f9 100644 --- a/python/grass/script/core.py +++ b/python/grass/script/core.py @@ -573,7 +573,7 @@ def parse_command(*args, **kwargs): """ def parse_csv(result): - return list(csv.reader(io.StringIO(result))) + return list(csv.DictReader(io.StringIO(result))) parse = None parse_args = {} From 93601f8c41e199a5e1d9cb35486c2abb5c76bc39 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Mon, 6 May 2024 10:07:06 -0400 Subject: [PATCH 3/4] add tests --- .../test_start_command_functions_nc.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/python/grass/script/testsuite/test_start_command_functions_nc.py b/python/grass/script/testsuite/test_start_command_functions_nc.py index 22e2f4f5d21..032560f56d1 100644 --- a/python/grass/script/testsuite/test_start_command_functions_nc.py +++ b/python/grass/script/testsuite/test_start_command_functions_nc.py @@ -3,7 +3,8 @@ from grass.gunittest.case import TestCase from grass.gunittest.main import test -from grass.script.core import start_command, PIPE +from grass.script.core import parse_command, start_command, PIPE +from grass.script.utils import parse_key_val LOCATION = "nc" @@ -49,5 +50,38 @@ def test_multiple_underscores(self): self.assertIn(b"raster", stderr) +class TestParseCommand(TestCase): + """Tests parse_command""" + + def test_parse_default(self): + result = parse_command("r.info", map="elevation", flags="g") + self.assertTrue( + isinstance(result, dict) and isinstance(result.get("north"), str) + ) + result_2 = parse_command("r.info", map="elevation", flags="g", delimiter="=") + self.assertDictEqual(result, result_2) + result_3 = parse_command( + "r.info", map="elevation", flags="g", parse=(parse_key_val, {"sep": "="}) + ) + self.assertDictEqual(result, result_3) + + def test_parse_format_json(self): + result = parse_command( + "r.what", map="elevation", coordinates=(640000, 220000), format="json" + ) + self.assertTrue( + isinstance(result, list) + and isinstance(result[0].get("easting"), (int, float)) + ) + result = parse_command("v.db.select", map="zipcodes", format="json") + + def test_parse_format_csv(self): + reference = parse_command("v.db.select", map="zipcodes", format="json")[ + "records" + ] + result = parse_command("v.db.select", map="zipcodes", format="csv") + self.assertListEqual(list(reference[0].keys()), list(result[0].keys())) + + if __name__ == "__main__": test() From a79420260dc0fa3ea772744fc2fe70d6608e3dfc Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Tue, 7 May 2024 14:49:12 -0400 Subject: [PATCH 4/4] remove unnecessary code in test --- python/grass/script/testsuite/test_start_command_functions_nc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/grass/script/testsuite/test_start_command_functions_nc.py b/python/grass/script/testsuite/test_start_command_functions_nc.py index 032560f56d1..4ad5d46f6e5 100644 --- a/python/grass/script/testsuite/test_start_command_functions_nc.py +++ b/python/grass/script/testsuite/test_start_command_functions_nc.py @@ -73,7 +73,6 @@ def test_parse_format_json(self): isinstance(result, list) and isinstance(result[0].get("easting"), (int, float)) ) - result = parse_command("v.db.select", map="zipcodes", format="json") def test_parse_format_csv(self): reference = parse_command("v.db.select", map="zipcodes", format="json")[