diff --git a/alt_e2eshark/reports/test_add.json b/alt_e2eshark/reports/test_add.json new file mode 100644 index 00000000..aaeb8d66 --- /dev/null +++ b/alt_e2eshark/reports/test_add.json @@ -0,0 +1,5 @@ +{ + "test_add": "PASS", + "test_add_bcast": "PASS", + "test_add_uint8": "PASS" +} \ No newline at end of file diff --git a/alt_e2eshark/reports/test_add.md b/alt_e2eshark/reports/test_add.md new file mode 100644 index 00000000..485e31ab --- /dev/null +++ b/alt_e2eshark/reports/test_add.md @@ -0,0 +1,26 @@ +## Summary + +|Stage|Count| +|--|--| +| Total | 3 | +| PASS | 3 | +| Numerics | 0 | +| results-summary | 0 | +| postprocessing | 0 | +| compiled_inference | 0 | +| native_inference | 0 | +| construct_inputs | 0 | +| compilation | 0 | +| preprocessing | 0 | +| import_model | 0 | +| setup | 0 | + +## Test Run Detail +Test was run with the following arguments: +Namespace(device='local-task', backend='llvm-cpu', iree_compile_args=None, mode='onnx-iree', torchtolinalg=False, stages=None, skip_stages=None, load_inputs=False, groups='all', test_filter='test_add', tolerance=None, verbose=True, rundirectory='test-run', no_artifacts=False, report=True, report_file='reports/test_add.md') + +| Test | Exit Status | Notes | +|--|--|--| +| test_add | PASS | | +| test_add_bcast | PASS | | +| test_add_uint8 | PASS | | diff --git a/alt_e2eshark/reports/test_conv.json b/alt_e2eshark/reports/test_conv.json new file mode 100644 index 00000000..cfe58808 --- /dev/null +++ b/alt_e2eshark/reports/test_conv.json @@ -0,0 +1,17 @@ +{ + "test_conv_with_autopad_same": "compilation", + "test_conv_with_strides_and_asymmetric_padding": "PASS", + "test_conv_with_strides_no_padding": "PASS", + "test_conv_with_strides_padding": "PASS", + "test_convinteger_with_padding": "PASS", + "test_convinteger_without_padding": "PASS", + "test_convtranspose": "PASS", + "test_convtranspose_1d": "PASS", + "test_convtranspose_3d": "PASS", + "test_convtranspose_autopad_same": "compilation", + "test_convtranspose_dilations": "PASS", + "test_convtranspose_kernel_shape": "compilation", + "test_convtranspose_output_shape": "compilation", + "test_convtranspose_pad": "PASS", + "test_convtranspose_pads": "PASS" +} \ No newline at end of file diff --git a/alt_e2eshark/reports/test_conv.md b/alt_e2eshark/reports/test_conv.md new file mode 100644 index 00000000..9f7a95eb --- /dev/null +++ b/alt_e2eshark/reports/test_conv.md @@ -0,0 +1,38 @@ +## Summary + +|Stage|Count| +|--|--| +| Total | 15 | +| PASS | 11 | +| Numerics | 0 | +| results-summary | 0 | +| postprocessing | 0 | +| compiled_inference | 0 | +| native_inference | 0 | +| construct_inputs | 0 | +| compilation | 4 | +| preprocessing | 0 | +| import_model | 0 | +| setup | 0 | + +## Test Run Detail +Test was run with the following arguments: +Namespace(device='local-task', backend='llvm-cpu', iree_compile_args=None, mode='onnx-iree', torchtolinalg=False, stages=None, skip_stages=None, load_inputs=False, groups='all', test_filter='test_conv', tolerance=None, verbose=True, rundirectory='test-run', no_artifacts=False, report=True, report_file='reports/test_conv.md') + +| Test | Exit Status | Notes | +|--|--|--| +| test_conv_with_autopad_same | compilation | | +| test_conv_with_strides_and_asymmetric_padding | PASS | | +| test_conv_with_strides_no_padding | PASS | | +| test_conv_with_strides_padding | PASS | | +| test_convinteger_with_padding | PASS | | +| test_convinteger_without_padding | PASS | | +| test_convtranspose | PASS | | +| test_convtranspose_1d | PASS | | +| test_convtranspose_3d | PASS | | +| test_convtranspose_autopad_same | compilation | | +| test_convtranspose_dilations | PASS | | +| test_convtranspose_kernel_shape | compilation | | +| test_convtranspose_output_shape | compilation | | +| test_convtranspose_pad | PASS | | +| test_convtranspose_pads | PASS | | diff --git a/alt_e2eshark/reports/test_conv_and_add.json b/alt_e2eshark/reports/test_conv_and_add.json new file mode 100644 index 00000000..63f0d1c2 --- /dev/null +++ b/alt_e2eshark/reports/test_conv_and_add.json @@ -0,0 +1,20 @@ +{ + "test_add": "PASS", + "test_add_bcast": "PASS", + "test_add_uint8": "PASS", + "test_conv_with_autopad_same": "compilation", + "test_conv_with_strides_and_asymmetric_padding": "PASS", + "test_conv_with_strides_no_padding": "PASS", + "test_conv_with_strides_padding": "PASS", + "test_convinteger_with_padding": "PASS", + "test_convinteger_without_padding": "PASS", + "test_convtranspose": "PASS", + "test_convtranspose_1d": "PASS", + "test_convtranspose_3d": "PASS", + "test_convtranspose_autopad_same": "compilation", + "test_convtranspose_dilations": "PASS", + "test_convtranspose_kernel_shape": "compilation", + "test_convtranspose_output_shape": "compilation", + "test_convtranspose_pad": "PASS", + "test_convtranspose_pads": "PASS" +} \ No newline at end of file diff --git a/alt_e2eshark/reports/test_conv_and_add.md b/alt_e2eshark/reports/test_conv_and_add.md new file mode 100644 index 00000000..f88829bf --- /dev/null +++ b/alt_e2eshark/reports/test_conv_and_add.md @@ -0,0 +1,41 @@ +## Summary + +|Stage|Count| +|--|--| +| Total | 18 | +| PASS | 14 | +| Numerics | 0 | +| results-summary | 0 | +| postprocessing | 0 | +| compiled_inference | 0 | +| native_inference | 0 | +| construct_inputs | 0 | +| compilation | 4 | +| preprocessing | 0 | +| import_model | 0 | +| setup | 0 | + +## Test Run Detail +Test was run with the following arguments: +Namespace(sources=['reports/test_add.json', 'reports/test_conv.json'], output='reports/test_conv_and_add.json', report=True, report_file='reports/test_conv_and_add.md') + +| Test | Exit Status | Notes | +|--|--|--| +| test_add | PASS | | +| test_add_bcast | PASS | | +| test_add_uint8 | PASS | | +| test_conv_with_autopad_same | compilation | | +| test_conv_with_strides_and_asymmetric_padding | PASS | | +| test_conv_with_strides_no_padding | PASS | | +| test_conv_with_strides_padding | PASS | | +| test_convinteger_with_padding | PASS | | +| test_convinteger_without_padding | PASS | | +| test_convtranspose | PASS | | +| test_convtranspose_1d | PASS | | +| test_convtranspose_3d | PASS | | +| test_convtranspose_autopad_same | compilation | | +| test_convtranspose_dilations | PASS | | +| test_convtranspose_kernel_shape | compilation | | +| test_convtranspose_output_shape | compilation | | +| test_convtranspose_pad | PASS | | +| test_convtranspose_pads | PASS | | diff --git a/alt_e2eshark/run.py b/alt_e2eshark/run.py index 4cbff4ec..368ddd77 100644 --- a/alt_e2eshark/run.py +++ b/alt_e2eshark/run.py @@ -28,7 +28,7 @@ # import backends from e2e_testing.backends import SimpleIREEBackend, OnnxrtIreeEpBackend -from utils.report import generate_report +from utils.report import generate_report, save_dict ALL_STAGES = [ "setup", @@ -107,7 +107,9 @@ def main(args): ) if args.report: - generate_report(args, stages, test_list, status_dict) + generate_report(args, stages, status_dict) + json_save_to = str(Path(args.report_file).parent.joinpath(Path(args.report_file).stem + ".json")) + save_dict(status_dict, json_save_to) def run_tests( @@ -190,6 +192,21 @@ def run_tests( golden_outputs_raw = inst.forward(inputs) golden_outputs_raw.save_to(log_dir + "golden_output") + # get inputs from inst + curr_stage = "construct_inputs" + if curr_stage in stages: + if load_inputs: + inputs = inst.load_inputs(log_dir) + else: + inputs = inst.construct_inputs() + inputs.save_to(log_dir + "input") + + # run native inference + curr_stage = "native_inference" + if curr_stage in stages: + golden_outputs_raw = inst.forward(inputs) + golden_outputs_raw.save_to(log_dir + "golden_output") + # run inference with the compiled module curr_stage = "compiled_inference" if curr_stage in stages: diff --git a/alt_e2eshark/utils/merge_dicts.py b/alt_e2eshark/utils/merge_dicts.py new file mode 100644 index 00000000..153d1951 --- /dev/null +++ b/alt_e2eshark/utils/merge_dicts.py @@ -0,0 +1,63 @@ +import json +import argparse +from report import generate_report, save_dict +from pathlib import Path +import sys + +TEST_DIR = str(Path(__file__).parents[1]) +sys.path.append(TEST_DIR) +from run import ALL_STAGES + + +def _get_argparse(): + msg = "A script for loading two or more dictionary jsons and merging them." + parser = argparse.ArgumentParser(prog="merge_dicts.py", description=msg, epilog="") + parser.add_argument( + "-s", + "--sources", + nargs="*", + required=True, + help="specify paths to source files for merging", + ) + parser.add_argument( + "-o", + "--output", + help="specify output filepath", + ) + parser.add_argument( + "-r", + "--report", + action="store_true", + default=False, + help="set this flag to generate a report from the merged status dicts", + ) + parser.add_argument( + "-f", + "--report-file", + default="report.md", + help="specify report filepath for merged status dicts", + ) + return parser + + +def main(args): + sources = args.sources + print(sources) + s0 = load(sources[0]) + for s in sources[1:]: + s0.update(load(s)) + if args.output: + save_dict(s0, args.output) + if args.report_file: + generate_report(args, ALL_STAGES, s0) + + +def load(pathname): + with open(pathname) as contents: + loaded_dict = json.load(contents) + return loaded_dict + + +if __name__ == "__main__": + parser = _get_argparse() + main(parser.parse_args()) diff --git a/alt_e2eshark/utils/report.py b/alt_e2eshark/utils/report.py index 51318c75..e4f454ec 100644 --- a/alt_e2eshark/utils/report.py +++ b/alt_e2eshark/utils/report.py @@ -3,8 +3,15 @@ # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +import json +import io -def generate_report(args, stages, test_list, status_dict): +def save_dict(status_dict, status_dict_json): + with io.open(status_dict_json, "w", encoding="utf8") as outfile: + dict_str = json.dumps(status_dict, indent=4, sort_keys=True, separators=(',',': '), ensure_ascii=False) + outfile.write(dict_str) + +def generate_report(args, stages, status_dict): """generates a markdown report for a test-run""" # set up report summary @@ -16,21 +23,18 @@ def generate_report(args, stages, test_list, status_dict): for (key, value) in status_dict.items(): counts[value] += 1 results_str = "## Summary\n\n|Stage|Count|\n|--|--|\n" - results_str += f"| Total | {len(test_list)} |\n" + results_str += f"| Total | {len(status_dict.keys())} |\n" for (key, value) in counts.items(): results_str += f"| {key} | {value} |\n" - + # set up report detail - report_string = f"\n## Test Run Detail \n Test was run with the following arguments:\n{args}\n\n" + report_string = f"\n## Test Run Detail \nTest was run with the following arguments:\n{args}\n\n" report_string += "| Test | Exit Status | Notes |\n" report_string += "|--|--|--|\n" for (key, value) in status_dict.items(): report_string += f"| {key} | {value} | |\n" - + # get a report file and write to it - report_file = "report.md" - if args.report_file: - report_file = args.report_file - with open(report_file, "w") as file: + with open(args.report_file, "w") as file: file.write(results_str) file.write(report_string) \ No newline at end of file