[alt] add a report merging tool (#331)

One of the items for
<#300>

alt_e2eshark/reports/test_add.json

@@ -0,0 +1,5 @@
+{
+    "test_add": "PASS",
+    "test_add_bcast": "PASS",
+    "test_add_uint8": "PASS"
+}

alt_e2eshark/reports/test_add.md

@@ -0,0 +1,26 @@
+## Summary
+
+|Stage|Count|
+|--|--|
+| Total | 3 |
+| PASS | 3 |
+| Numerics | 0 |
+| results-summary | 0 |
+| postprocessing | 0 |
+| compiled_inference | 0 |
+| native_inference | 0 |
+| construct_inputs | 0 |
+| compilation | 0 |
+| preprocessing | 0 |
+| import_model | 0 |
+| setup | 0 |
+
+## Test Run Detail 
+Test was run with the following arguments:
+Namespace(device='local-task', backend='llvm-cpu', iree_compile_args=None, mode='onnx-iree', torchtolinalg=False, stages=None, skip_stages=None, load_inputs=False, groups='all', test_filter='test_add', tolerance=None, verbose=True, rundirectory='test-run', no_artifacts=False, report=True, report_file='reports/test_add.md')
+
+| Test | Exit Status | Notes |
+|--|--|--|
+| test_add | PASS | |
+| test_add_bcast | PASS | |
+| test_add_uint8 | PASS | |

alt_e2eshark/reports/test_conv.json

@@ -0,0 +1,17 @@
+{
+    "test_conv_with_autopad_same": "compilation",
+    "test_conv_with_strides_and_asymmetric_padding": "PASS",
+    "test_conv_with_strides_no_padding": "PASS",
+    "test_conv_with_strides_padding": "PASS",
+    "test_convinteger_with_padding": "PASS",
+    "test_convinteger_without_padding": "PASS",
+    "test_convtranspose": "PASS",
+    "test_convtranspose_1d": "PASS",
+    "test_convtranspose_3d": "PASS",
+    "test_convtranspose_autopad_same": "compilation",
+    "test_convtranspose_dilations": "PASS",
+    "test_convtranspose_kernel_shape": "compilation",
+    "test_convtranspose_output_shape": "compilation",
+    "test_convtranspose_pad": "PASS",
+    "test_convtranspose_pads": "PASS"
+}

alt_e2eshark/reports/test_conv.md

@@ -0,0 +1,38 @@
+## Summary
+
+|Stage|Count|
+|--|--|
+| Total | 15 |
+| PASS | 11 |
+| Numerics | 0 |
+| results-summary | 0 |
+| postprocessing | 0 |
+| compiled_inference | 0 |
+| native_inference | 0 |
+| construct_inputs | 0 |
+| compilation | 4 |
+| preprocessing | 0 |
+| import_model | 0 |
+| setup | 0 |
+
+## Test Run Detail 
+Test was run with the following arguments:
+Namespace(device='local-task', backend='llvm-cpu', iree_compile_args=None, mode='onnx-iree', torchtolinalg=False, stages=None, skip_stages=None, load_inputs=False, groups='all', test_filter='test_conv', tolerance=None, verbose=True, rundirectory='test-run', no_artifacts=False, report=True, report_file='reports/test_conv.md')
+
+| Test | Exit Status | Notes |
+|--|--|--|
+| test_conv_with_autopad_same | compilation | |
+| test_conv_with_strides_and_asymmetric_padding | PASS | |
+| test_conv_with_strides_no_padding | PASS | |
+| test_conv_with_strides_padding | PASS | |
+| test_convinteger_with_padding | PASS | |
+| test_convinteger_without_padding | PASS | |
+| test_convtranspose | PASS | |
+| test_convtranspose_1d | PASS | |
+| test_convtranspose_3d | PASS | |
+| test_convtranspose_autopad_same | compilation | |
+| test_convtranspose_dilations | PASS | |
+| test_convtranspose_kernel_shape | compilation | |
+| test_convtranspose_output_shape | compilation | |
+| test_convtranspose_pad | PASS | |
+| test_convtranspose_pads | PASS | |

alt_e2eshark/reports/test_conv_and_add.json

@@ -0,0 +1,20 @@
+{
+    "test_add": "PASS",
+    "test_add_bcast": "PASS",
+    "test_add_uint8": "PASS",
+    "test_conv_with_autopad_same": "compilation",
+    "test_conv_with_strides_and_asymmetric_padding": "PASS",
+    "test_conv_with_strides_no_padding": "PASS",
+    "test_conv_with_strides_padding": "PASS",
+    "test_convinteger_with_padding": "PASS",
+    "test_convinteger_without_padding": "PASS",
+    "test_convtranspose": "PASS",
+    "test_convtranspose_1d": "PASS",
+    "test_convtranspose_3d": "PASS",
+    "test_convtranspose_autopad_same": "compilation",
+    "test_convtranspose_dilations": "PASS",
+    "test_convtranspose_kernel_shape": "compilation",
+    "test_convtranspose_output_shape": "compilation",
+    "test_convtranspose_pad": "PASS",
+    "test_convtranspose_pads": "PASS"
+}

alt_e2eshark/reports/test_conv_and_add.md

@@ -0,0 +1,41 @@
+## Summary
+
+|Stage|Count|
+|--|--|
+| Total | 18 |
+| PASS | 14 |
+| Numerics | 0 |
+| results-summary | 0 |
+| postprocessing | 0 |
+| compiled_inference | 0 |
+| native_inference | 0 |
+| construct_inputs | 0 |
+| compilation | 4 |
+| preprocessing | 0 |
+| import_model | 0 |
+| setup | 0 |
+
+## Test Run Detail 
+Test was run with the following arguments:
+Namespace(sources=['reports/test_add.json', 'reports/test_conv.json'], output='reports/test_conv_and_add.json', report=True, report_file='reports/test_conv_and_add.md')
+
+| Test | Exit Status | Notes |
+|--|--|--|
+| test_add | PASS | |
+| test_add_bcast | PASS | |
+| test_add_uint8 | PASS | |
+| test_conv_with_autopad_same | compilation | |
+| test_conv_with_strides_and_asymmetric_padding | PASS | |
+| test_conv_with_strides_no_padding | PASS | |
+| test_conv_with_strides_padding | PASS | |
+| test_convinteger_with_padding | PASS | |
+| test_convinteger_without_padding | PASS | |
+| test_convtranspose | PASS | |
+| test_convtranspose_1d | PASS | |
+| test_convtranspose_3d | PASS | |
+| test_convtranspose_autopad_same | compilation | |
+| test_convtranspose_dilations | PASS | |
+| test_convtranspose_kernel_shape | compilation | |
+| test_convtranspose_output_shape | compilation | |
+| test_convtranspose_pad | PASS | |
+| test_convtranspose_pads | PASS | |

alt_e2eshark/run.py

@@ -28,7 +28,7 @@
 
 # import backends
 from e2e_testing.backends import SimpleIREEBackend, OnnxrtIreeEpBackend
-from utils.report import generate_report
+from utils.report import generate_report, save_dict
 
 ALL_STAGES = [
     "setup",
@@ -107,7 +107,9 @@ def main(args):
     )
 
     if args.report:
-        generate_report(args, stages, test_list, status_dict)
+        generate_report(args, stages, status_dict)
+        json_save_to = str(Path(args.report_file).parent.joinpath(Path(args.report_file).stem + ".json"))
+        save_dict(status_dict, json_save_to)
 
 
 def run_tests(
@@ -190,6 +192,21 @@ def run_tests(
                 golden_outputs_raw = inst.forward(inputs)
                 golden_outputs_raw.save_to(log_dir + "golden_output")
 
+            # get inputs from inst
+            curr_stage = "construct_inputs"
+            if curr_stage in stages:
+                if load_inputs:
+                    inputs = inst.load_inputs(log_dir)
+                else:
+                    inputs = inst.construct_inputs()
+                    inputs.save_to(log_dir + "input")
+
+            # run native inference
+            curr_stage = "native_inference"
+            if curr_stage in stages:
+                golden_outputs_raw = inst.forward(inputs)
+                golden_outputs_raw.save_to(log_dir + "golden_output")
+
             # run inference with the compiled module
             curr_stage = "compiled_inference"
             if curr_stage in stages:

alt_e2eshark/utils/merge_dicts.py

@@ -0,0 +1,63 @@
+import json
+import argparse
+from report import generate_report, save_dict
+from pathlib import Path
+import sys
+
+TEST_DIR = str(Path(__file__).parents[1])
+sys.path.append(TEST_DIR)
+from run import ALL_STAGES
+
+
+def _get_argparse():
+    msg = "A script for loading two or more dictionary jsons and merging them."
+    parser = argparse.ArgumentParser(prog="merge_dicts.py", description=msg, epilog="")
+    parser.add_argument(
+        "-s",
+        "--sources",
+        nargs="*",
+        required=True,
+        help="specify paths to source files for merging",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="specify output filepath",
+    )
+    parser.add_argument(
+        "-r",
+        "--report",
+        action="store_true",
+        default=False,
+        help="set this flag to generate a report from the merged status dicts",
+    )
+    parser.add_argument(
+        "-f",
+        "--report-file",
+        default="report.md",
+        help="specify report filepath for merged status dicts",
+    )
+    return parser
+
+
+def main(args):
+    sources = args.sources
+    print(sources)
+    s0 = load(sources[0])
+    for s in sources[1:]:
+        s0.update(load(s))
+    if args.output:
+        save_dict(s0, args.output)
+    if args.report_file:
+        generate_report(args, ALL_STAGES, s0)
+
+
+def load(pathname):
+    with open(pathname) as contents:
+        loaded_dict = json.load(contents)
+    return loaded_dict
+
+
+if __name__ == "__main__":
+    parser = _get_argparse()
+    main(parser.parse_args())

alt_e2eshark/utils/report.py

@@ -3,8 +3,15 @@
 # Licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+import json
+import io
 
-def generate_report(args, stages, test_list, status_dict):
+def save_dict(status_dict, status_dict_json):
+    with io.open(status_dict_json, "w", encoding="utf8") as outfile:
+        dict_str = json.dumps(status_dict, indent=4, sort_keys=True, separators=(',',': '), ensure_ascii=False)
+        outfile.write(dict_str)
+
+def generate_report(args, stages, status_dict):
     """generates a markdown report for a test-run"""
 
     # set up report summary
@@ -16,21 +23,18 @@ def generate_report(args, stages, test_list, status_dict):
     for (key, value) in status_dict.items():
         counts[value] += 1
     results_str = "## Summary\n\n|Stage|Count|\n|--|--|\n"
-    results_str += f"| Total | {len(test_list)} |\n"
+    results_str += f"| Total | {len(status_dict.keys())} |\n"
     for (key, value) in counts.items():
         results_str += f"| {key} | {value} |\n"
-    
+
     # set up report detail
-    report_string = f"\n## Test Run Detail \n Test was run with the following arguments:\n{args}\n\n"
+    report_string = f"\n## Test Run Detail \nTest was run with the following arguments:\n{args}\n\n"
     report_string += "| Test | Exit Status | Notes |\n"
     report_string += "|--|--|--|\n"
     for (key, value) in status_dict.items():
         report_string += f"| {key} | {value} | |\n"
-    
+
     # get a report file and write to it 
-    report_file = "report.md"
-    if args.report_file:
-        report_file = args.report_file
-    with open(report_file, "w") as file:
+    with open(args.report_file, "w") as file:
         file.write(results_str)
         file.write(report_string)