From f7ed5f0f18789fa7df1e97a0fedbbea772b5b322 Mon Sep 17 00:00:00 2001 From: Rafael Folco Date: Tue, 12 Sep 2023 14:27:27 -0300 Subject: [PATCH] Add multibench support to all-in-one json Add benchmarks block to identify bench ids. The blockbreaker utility extracts the benchmark from the runfile and processes multibench config. The mv-params for multibench is extracted from the 'benchmarks' block. To keep backwards compatibility with the single bench mode, it is still possible to define the mv-params block from the top level of the JSON file. For both single bench and multibench on the new schema, use blockbreaker w/ --config mv-params and --benchmark . For single bench on the old schema, use --config mv-params and --index . This requires additional changes on the crucible code to fully support 'benchmarks' block from the json runfile, and then support it from the crucible cli. --- util/JSON/schema.json | 25 +++-- util/blockbreaker.py | 55 +++++++++-- .../JSON/input-multibench-k8s-remotehost.json | 98 ++++++++++--------- util/tests/test-blockbreaker.py | 4 +- 4 files changed, 122 insertions(+), 60 deletions(-) diff --git a/util/JSON/schema.json b/util/JSON/schema.json index 057da83d..a0030c2e 100644 --- a/util/JSON/schema.json +++ b/util/JSON/schema.json @@ -10,7 +10,8 @@ "enum": [ "2022.11.08", "2023.03.10", - "2023.06.06" + "2023.06.06", + "2023.09.12" ] } } @@ -76,18 +77,30 @@ } } ] + }, + "mv-params": { + "anyOf": [ + { + "type": "array", + "items": { + "minItems": 1, + "type": "object" + } + }, + { + "type": "object" + } + ] } }, "additionalProperties": false, "required": [ "name", - "ids" + "ids", + "mv-params" ] } } }, - "additionalProperties": false, - "required": [ - "mv-params" - ] + "additionalProperties": false } diff --git a/util/blockbreaker.py b/util/blockbreaker.py index 4d99de14..e3a3ce80 100755 --- a/util/blockbreaker.py +++ b/util/blockbreaker.py @@ -24,7 +24,7 @@ def process_options(): dest = "config", required = True, help = "Configuration type to get from the json file", - choices = [ "mv-params", "tool-params", "passthru-args", "tags", "endpoint" ]) + choices = [ "benchmarks", "mv-params", "tool-params", "passthru-args", "tags", "endpoint" ]) parser.add_argument("--index", dest = "index", @@ -32,6 +32,11 @@ def process_options(): default = 0, type = int) + parser.add_argument("--benchmark", + dest = "benchmark", + help = "Crucible benchmark block to extract data from (e.g. 'uperf')", + type = str) + args = parser.parse_args() return args @@ -205,6 +210,33 @@ def validate_schema(input_json, schema_file = None): return False return True +def get_bench_ids(json_obj, cfg): + """Get benchmark names and ids from the benchmarks block""" + stream="" + if json_obj is None: + return "" + try: + bench_count = len(json_obj[cfg]) + if bench_count > 0: + for idx in range(0, bench_count): + json_blk = json_obj[cfg][idx] + stream+=json_blk["name"]+":"+json_blk["ids"]+"," + # remove last "," + if len(stream)>0: + stream = stream[:-1] + except: + pass + + return stream + +def get_mv_params(input_json, name): + """Extract mv-params from the benchmarks block""" + try: + benchmark_blk = next(d for d in input_json['benchmarks'] if d.get('name', None) == name) + except: + return None + return benchmark_blk + def main(): """Main function of get-json-config.py tool""" @@ -217,12 +249,21 @@ def main(): if not validate_schema(input_json): return 1 - if args.config == "endpoint" or args.config == "tags": - # output is a stream of the endpoint or tags - output = json_to_stream(input_json, args.config, args.index) - else: - # output is a JSON object (e.g. mv-params, tool-params) - output = dump_json(input_json, args.config, args.index) + match args.config: + case "benchmarks": + output = get_bench_ids(input_json, args.config) + case "endpoint" | "tags": + # output is a stream of the endpoint or tags + output = json_to_stream(input_json, args.config, args.index) + case default: + if args.config == "mv-params": + # if '--benchmark' is specified, get mv-params from + # the mv-params object inside of the 'benchmarks' block + if args.benchmark is not None: + input_json = get_mv_params(input_json, args.benchmark) + # mv-params, tool-params, passthru-args + output = dump_json(input_json, args.config, args.index) + print(output) if __name__ == "__main__": diff --git a/util/tests/JSON/input-multibench-k8s-remotehost.json b/util/tests/JSON/input-multibench-k8s-remotehost.json index b162f16b..be4b9a27 100644 --- a/util/tests/JSON/input-multibench-k8s-remotehost.json +++ b/util/tests/JSON/input-multibench-k8s-remotehost.json @@ -1,55 +1,63 @@ { "benchmarks": [ - { "name": "oslat", "ids": "1-2" }, - { "name": "uperf", "ids": ["3-4","6-7"] }, - { "name": "iperf", "ids": "5" } - ], - "mv-params": [ { - "sets": [ - { - "params": [ - { "arg": "duration", "vals": [ "10" ], "role": "client" }, - { "arg": "rtprio", "vals": [ "1" ], "role": "client" } - ] - } - ] + "name": "oslat", + "ids": "1-2", + "mv-params": { + "sets": [ + { + "params": [ + { "arg": "duration", "vals": [ "10" ], "role": "client" }, + { "arg": "rtprio", "vals": [ "1" ], "role": "client" } + ] + } + ] + } }, { - "sets": [ - { - "params": [ - { "arg": "ifname", "vals": ["eth0"], "role": "server" }, - { "arg": "ipv", "vals": ["6"], "role": "all" }, - { "arg": "protocol", "vals": ["udp"], "id": "21" }, - { "arg": "bitrate", "vals": ["200M"], "id": "21" }, - { "arg": "length", "vals": ["256"], "id": "21" } - ] - } - ] + "name": "uperf", + "ids": ["3-4","6-7"], + "mv-params": { + "sets": [ + { + "params": [ + { "arg": "ifname", "vals": ["eth0"], "role": "server" }, + { "arg": "ipv", "vals": ["6"], "role": "all" }, + { "arg": "protocol", "vals": ["udp"], "id": "21" }, + { "arg": "bitrate", "vals": ["200M"], "id": "21" }, + { "arg": "length", "vals": ["256"], "id": "21" } + ] + } + ] + + } }, { - "global-options": [ - { - "name": "common-params", - "params": [ - { "arg": "duration", "vals": ["60"] }, - { "arg": "protocol", "vals": ["tcp"] }, - { "arg": "nthreads", "vals": ["1"] }, - { "arg": "ifname", "vals": ["eth0"], "role": "server" } - ] - } - ], - "sets": [ - { - "include": "common-params", - "params": [ - { "arg": "test-type", "vals": ["crr"], "id": "1" }, - { "arg": "wsize", "vals": ["512"], "id": "1" }, - { "arg": "rsize", "vals": ["2048"], "id": "1" } - ] - } - ] + "name": "iperf", + "ids": "5", + "mv-params": { + "global-options": [ + { + "name": "common-params", + "params": [ + { "arg": "duration", "vals": ["60"] }, + { "arg": "protocol", "vals": ["tcp"] }, + { "arg": "nthreads", "vals": ["1"] }, + { "arg": "ifname", "vals": ["eth0"], "role": "server" } + ] + } + ], + "sets": [ + { + "include": "common-params", + "params": [ + { "arg": "test-type", "vals": ["crr"], "id": "1" }, + { "arg": "wsize", "vals": ["512"], "id": "1" }, + { "arg": "rsize", "vals": ["2048"], "id": "1" } + ] + } + ] + } } ], "tool-params": [ diff --git a/util/tests/test-blockbreaker.py b/util/tests/test-blockbreaker.py index c5526fa1..8c201d27 100755 --- a/util/tests/test-blockbreaker.py +++ b/util/tests/test-blockbreaker.py @@ -169,11 +169,11 @@ def test_dump_json_mvparams_object(self, load_json_file): expected_output = self._load_file("output-oslat-k8s-mvparams.json") assert input_json == expected_output - """Test if dump_json returns the correct json block from mv-params idx 1""" + """Test if dump_json returns the correct json block from mv-params for uperf""" @pytest.mark.parametrize("load_json_file", [ "input-multibench-k8s-remotehost.json" ], indirect=True) def test_dump_json_mvparams_index1(self, load_json_file): assert type(load_json_file) == dict - input_json = blockbreaker.dump_json(load_json_file, "mv-params", 1) + input_json = blockbreaker.dump_json(load_json_file["benchmarks"][1], "mv-params", "uperf") expected_output = self._load_file("output-multibench-k8s-remotehost-mvparams1.json") assert input_json == expected_output