From a45c418a5ff256b76acbaa5bd6f9177d1bfbdfd6 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 03:41:58 -0700 Subject: [PATCH 01/11] try adding the 4 pytorch models with onnx export --- iree_tests/configs/models_cpu_llvm_task.json | 17 +-- .../configs/models_gpu_rocm_gfx90a.json | 20 ++-- .../{ => direct}/opt-125M/opt-125M.mlirbc | 0 .../opt-125M/real_weights_data_flags.txt | 0 .../opt-125M/splat_data_flags.txt | 0 .../models/{ => direct}/opt-125M/splats.irpa | 0 .../{ => direct}/opt-125M/test_cases.json | 0 .../resnet50/real_weights_data_flags.txt | 0 .../{ => direct}/resnet50/resnet50.mlirbc | 0 .../resnet50/splat_data_flags.txt | 0 .../models/{ => direct}/resnet50/splats.irpa | 0 .../{ => direct}/resnet50/test_cases.json | 0 .../models/onnx-export/mit-b0/model.mlirbc | 3 + .../mit-b0/real_weights_data_flags.txt | 3 + .../models/onnx-export/mit-b0/test_cases.json | 13 ++ .../real_weights_data_flags.txt | 2 + .../mobilebert-uncased/test_cases.json | 14 +++ .../t5-base/real_weights_data_flags.txt | 53 +++++++++ .../onnx-export/t5-base/test_cases.json | 65 ++++++++++ .../t5-large/real_weights_data_flags.txt | 101 ++++++++++++++++ .../onnx-export/t5-large/test_cases.json | 112 ++++++++++++++++++ 21 files changed, 384 insertions(+), 19 deletions(-) rename iree_tests/pytorch/models/{ => direct}/opt-125M/opt-125M.mlirbc (100%) rename iree_tests/pytorch/models/{ => direct}/opt-125M/real_weights_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => direct}/opt-125M/splat_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => direct}/opt-125M/splats.irpa (100%) rename iree_tests/pytorch/models/{ => direct}/opt-125M/test_cases.json (100%) rename iree_tests/pytorch/models/{ => direct}/resnet50/real_weights_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => direct}/resnet50/resnet50.mlirbc (100%) rename iree_tests/pytorch/models/{ => direct}/resnet50/splat_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => direct}/resnet50/splats.irpa (100%) rename iree_tests/pytorch/models/{ => direct}/resnet50/test_cases.json (100%) create mode 100644 iree_tests/pytorch/models/onnx-export/mit-b0/model.mlirbc create mode 100644 iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt create mode 100644 iree_tests/pytorch/models/onnx-export/mit-b0/test_cases.json create mode 100644 iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt create mode 100644 iree_tests/pytorch/models/onnx-export/mobilebert-uncased/test_cases.json create mode 100644 iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt create mode 100644 iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json create mode 100644 iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt create mode 100644 iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json diff --git a/iree_tests/configs/models_cpu_llvm_task.json b/iree_tests/configs/models_cpu_llvm_task.json index f02bf5827..a2c00ec04 100644 --- a/iree_tests/configs/models_cpu_llvm_task.json +++ b/iree_tests/configs/models_cpu_llvm_task.json @@ -7,15 +7,16 @@ "iree_run_module_flags": [ "--device=local-task" ], - "skip_compile_tests": [ - "pytorch/models/sdxl-scheduled-unet-3-tank", - "pytorch/models/sdxl-vae-decode-tank", - "pytorch/models/sdxl-prompt-encoder-tank" - ], + "skip_compile_tests": [], "skip_run_tests": [], "expected_compile_failures": [ - "pytorch/models/opt-125M", // TODO(#17344): need to regenerate .mlirbc - "pytorch/models/resnet50", // TODO(#17344): need to regenerate .mlirbc + "pytorch/models/direct/opt-125M", // TODO(#17344): need to regenerate .mlirbc + "pytorch/models/direct/resnet50", // TODO(#17344): need to regenerate .mlirbc ], - "expected_run_failures": [] + "expected_run_failures": [ + "pytorch/models/onnx-export/mit-b0", + "pytorch/models/direct/mobilebert-uncased", + "pytorch/models/direct/t5-base", + "pytorch/models/direct/t5-large" + ] } diff --git a/iree_tests/configs/models_gpu_rocm_gfx90a.json b/iree_tests/configs/models_gpu_rocm_gfx90a.json index 85c726dab..2ce65d591 100644 --- a/iree_tests/configs/models_gpu_rocm_gfx90a.json +++ b/iree_tests/configs/models_gpu_rocm_gfx90a.json @@ -2,25 +2,23 @@ "config_name": "gpu_rocm", "iree_compile_flags" : [ "--iree-hal-target-backends=rocm", - "--iree-rocm-target-chip=gfx90a", - "--iree-opt-const-eval=false", - "--iree-codegen-transform-dialect-library=${IREE_TEST_PATH_EXTENSION}/attention_and_matmul_spec.mlir" + "--iree-rocm-target-chip=gfx90a" ], "iree_run_module_flags": [ "--device=hip" ], - "skip_compile_tests": [ - "pytorch/models/sdxl-scheduled-unet-3-tank", - "pytorch/models/sdxl-vae-decode-tank", - "pytorch/models/sdxl-prompt-encoder-tank" - ], + "skip_compile_tests": [], "skip_run_tests": [], "expected_compile_failures": [ - "pytorch/models/opt-125M", // TODO(#17344): need to regenerate .mlirbc - "pytorch/models/resnet50", + "pytorch/models/direct/opt-125M", // TODO(#17344): need to regenerate .mlirbc + "pytorch/models/direct/resnet50", // error: 'builtin.module' op failed to run transform dialect passes // (might need to drop the iree-codegen-transform-dialect-library flag) - "sharktank/llama/open-llama-3b-v2-f16" + "sharktank/llama/open-llama-3b-v2-f16", + "pytorch/models/onnx-export/mit-b0", + "pytorch/models/onnx-export/mobilebert-uncased", + "pytorch/models/onnx-export/t5-base", + "pytorch/models/onnx-export/t5-large" ], "expected_run_failures": [] } diff --git a/iree_tests/pytorch/models/opt-125M/opt-125M.mlirbc b/iree_tests/pytorch/models/direct/opt-125M/opt-125M.mlirbc similarity index 100% rename from iree_tests/pytorch/models/opt-125M/opt-125M.mlirbc rename to iree_tests/pytorch/models/direct/opt-125M/opt-125M.mlirbc diff --git a/iree_tests/pytorch/models/opt-125M/real_weights_data_flags.txt b/iree_tests/pytorch/models/direct/opt-125M/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/opt-125M/real_weights_data_flags.txt rename to iree_tests/pytorch/models/direct/opt-125M/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/opt-125M/splat_data_flags.txt b/iree_tests/pytorch/models/direct/opt-125M/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/opt-125M/splat_data_flags.txt rename to iree_tests/pytorch/models/direct/opt-125M/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/opt-125M/splats.irpa b/iree_tests/pytorch/models/direct/opt-125M/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/opt-125M/splats.irpa rename to iree_tests/pytorch/models/direct/opt-125M/splats.irpa diff --git a/iree_tests/pytorch/models/opt-125M/test_cases.json b/iree_tests/pytorch/models/direct/opt-125M/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/opt-125M/test_cases.json rename to iree_tests/pytorch/models/direct/opt-125M/test_cases.json diff --git a/iree_tests/pytorch/models/resnet50/real_weights_data_flags.txt b/iree_tests/pytorch/models/direct/resnet50/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/resnet50/real_weights_data_flags.txt rename to iree_tests/pytorch/models/direct/resnet50/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/resnet50/resnet50.mlirbc b/iree_tests/pytorch/models/direct/resnet50/resnet50.mlirbc similarity index 100% rename from iree_tests/pytorch/models/resnet50/resnet50.mlirbc rename to iree_tests/pytorch/models/direct/resnet50/resnet50.mlirbc diff --git a/iree_tests/pytorch/models/resnet50/splat_data_flags.txt b/iree_tests/pytorch/models/direct/resnet50/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/resnet50/splat_data_flags.txt rename to iree_tests/pytorch/models/direct/resnet50/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/resnet50/splats.irpa b/iree_tests/pytorch/models/direct/resnet50/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/resnet50/splats.irpa rename to iree_tests/pytorch/models/direct/resnet50/splats.irpa diff --git a/iree_tests/pytorch/models/resnet50/test_cases.json b/iree_tests/pytorch/models/direct/resnet50/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/resnet50/test_cases.json rename to iree_tests/pytorch/models/direct/resnet50/test_cases.json diff --git a/iree_tests/pytorch/models/onnx-export/mit-b0/model.mlirbc b/iree_tests/pytorch/models/onnx-export/mit-b0/model.mlirbc new file mode 100644 index 000000000..5a69772e7 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/mit-b0/model.mlirbc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd78ef5b57753bbc18555f93e52ec3a83b26010421b4d7961e94250d5ec65263 +size 14341754 diff --git a/iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt new file mode 100644 index 000000000..56855264e --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt @@ -0,0 +1,3 @@ +--input=1x3x512x512xf32=@inference_input.0.bin +--expected_output=1x3x512x512xf32=@inference_output.0.bin +--expected_f32_threshold=0.01f diff --git a/iree_tests/pytorch/models/onnx-export/mit-b0/test_cases.json b/iree_tests/pytorch/models/onnx-export/mit-b0/test_cases.json new file mode 100644 index 000000000..3a1f08249 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/mit-b0/test_cases.json @@ -0,0 +1,13 @@ +{ + "file_format": "test_cases_v0", + "test_cases": [ + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_files": [ + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/mit-b0/inference_input.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/mit-b0/inference_output.0.bin" + ] + } + ] +} diff --git a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt new file mode 100644 index 000000000..41ba48aa1 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt @@ -0,0 +1,2 @@ +--input=1x128xi64=@inference_input.0.bin +--expected_output=1x128xi64=@inference_output.0.bin diff --git a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/test_cases.json b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/test_cases.json new file mode 100644 index 000000000..2ca25c7b4 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/test_cases.json @@ -0,0 +1,14 @@ +{ + "file_format": "test_cases_v0", + "test_cases": [ + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_files": [ + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/mobilebert-uncased/inference_input.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/mobilebert-uncased/inference_output.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/mobilebert-uncased/model.mlirbc" + ] + } + ] +} diff --git a/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt new file mode 100644 index 000000000..b089ce101 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt @@ -0,0 +1,53 @@ +--input=1x512xi64=@inference_input.0.bin +--input=1x512xf32=@inference_input.1.bin +--input=1x512xi64=@inference_input.2.bin +--expected_output=1x512xi64=@inference_output.0.bin +--expected_output=1x512xi64=@inference_output.1.bin +--expected_output=1x512xi64=@inference_output.2.bin +--expected_output=1x512xi64=@inference_output.3.bin +--expected_output=1x512xi64=@inference_output.4.bin +--expected_output=1x512xi64=@inference_output.5.bin +--expected_output=1x512xi64=@inference_output.6.bin +--expected_output=1x512xi64=@inference_output.7.bin +--expected_output=1x512xi64=@inference_output.8.bin +--expected_output=1x512xi64=@inference_output.9.bin +--expected_output=1x512xi64=@inference_output.10.bin +--expected_output=1x512xi64=@inference_output.11.bin +--expected_output=1x512xi64=@inference_output.12.bin +--expected_output=1x512xi64=@inference_output.13.bin +--expected_output=1x512xi64=@inference_output.14.bin +--expected_output=1x512xi64=@inference_output.15.bin +--expected_output=1x512xi64=@inference_output.16.bin +--expected_output=1x512xi64=@inference_output.17.bin +--expected_output=1x512xi64=@inference_output.18.bin +--expected_output=1x512xi64=@inference_output.19.bin +--expected_output=1x512xi64=@inference_output.20.bin +--expected_output=1x512xi64=@inference_output.21.bin +--expected_output=1x512xi64=@inference_output.22.bin +--expected_output=1x512xi64=@inference_output.23.bin +--expected_output=1x512xi64=@inference_output.24.bin +--expected_output=1x512xi64=@inference_output.25.bin +--expected_output=1x512xi64=@inference_output.26.bin +--expected_output=1x512xi64=@inference_output.27.bin +--expected_output=1x512xi64=@inference_output.28.bin +--expected_output=1x512xi64=@inference_output.29.bin +--expected_output=1x512xi64=@inference_output.30.bin +--expected_output=1x512xi64=@inference_output.31.bin +--expected_output=1x512xi64=@inference_output.32.bin +--expected_output=1x512xi64=@inference_output.33.bin +--expected_output=1x512xi64=@inference_output.34.bin +--expected_output=1x512xi64=@inference_output.35.bin +--expected_output=1x512xi64=@inference_output.36.bin +--expected_output=1x512xi64=@inference_output.37.bin +--expected_output=1x512xi64=@inference_output.38.bin +--expected_output=1x512xi64=@inference_output.39.bin +--expected_output=1x512xi64=@inference_output.40.bin +--expected_output=1x512xi64=@inference_output.41.bin +--expected_output=1x512xi64=@inference_output.42.bin +--expected_output=1x512xi64=@inference_output.43.bin +--expected_output=1x512xi64=@inference_output.44.bin +--expected_output=1x512xi64=@inference_output.45.bin +--expected_output=1x512xi64=@inference_output.46.bin +--expected_output=1x512xi64=@inference_output.47.bin +--expected_output=1x512xi64=@inference_output.48.bin +--expected_output=1x512xi64=@inference_output.49.bin diff --git a/iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json b/iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json new file mode 100644 index 000000000..4f371f843 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json @@ -0,0 +1,65 @@ +{ + "file_format": "test_cases_v0", + "test_cases": [ + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_files": [ + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_input.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_input.1.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_input.2.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.1.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.2.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.3.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.4.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.5.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.6.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.7.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.8.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.9.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.10.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.11.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.12.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.13.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.14.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.15.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.16.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.17.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.18.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.19.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.20.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.21.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.22.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.23.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.24.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.25.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.26.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.27.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.28.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.29.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.30.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.31.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.32.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.33.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.34.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.35.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.36.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.37.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.38.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.39.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.40.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.41.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.42.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.43.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.44.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.45.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.46.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.47.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.48.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.49.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/mobilebert-uncased/model.mlirbc" + ] + } + ] +} diff --git a/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt new file mode 100644 index 000000000..52324c6b0 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt @@ -0,0 +1,101 @@ +--input=1x512xi64=@inference_input.0.bin +--input=1x512xf32=@inference_input.1.bin +--input=1x512xi64=@inference_input.2.bin +--expected_output=1x512xi64=@inference_output.0.bin +--expected_output=1x512xi64=@inference_output.1.bin +--expected_output=1x512xi64=@inference_output.2.bin +--expected_output=1x512xi64=@inference_output.3.bin +--expected_output=1x512xi64=@inference_output.4.bin +--expected_output=1x512xi64=@inference_output.5.bin +--expected_output=1x512xi64=@inference_output.6.bin +--expected_output=1x512xi64=@inference_output.7.bin +--expected_output=1x512xi64=@inference_output.8.bin +--expected_output=1x512xi64=@inference_output.9.bin +--expected_output=1x512xi64=@inference_output.10.bin +--expected_output=1x512xi64=@inference_output.11.bin +--expected_output=1x512xi64=@inference_output.12.bin +--expected_output=1x512xi64=@inference_output.13.bin +--expected_output=1x512xi64=@inference_output.14.bin +--expected_output=1x512xi64=@inference_output.15.bin +--expected_output=1x512xi64=@inference_output.16.bin +--expected_output=1x512xi64=@inference_output.17.bin +--expected_output=1x512xi64=@inference_output.18.bin +--expected_output=1x512xi64=@inference_output.19.bin +--expected_output=1x512xi64=@inference_output.20.bin +--expected_output=1x512xi64=@inference_output.21.bin +--expected_output=1x512xi64=@inference_output.22.bin +--expected_output=1x512xi64=@inference_output.23.bin +--expected_output=1x512xi64=@inference_output.24.bin +--expected_output=1x512xi64=@inference_output.25.bin +--expected_output=1x512xi64=@inference_output.26.bin +--expected_output=1x512xi64=@inference_output.27.bin +--expected_output=1x512xi64=@inference_output.28.bin +--expected_output=1x512xi64=@inference_output.29.bin +--expected_output=1x512xi64=@inference_output.30.bin +--expected_output=1x512xi64=@inference_output.31.bin +--expected_output=1x512xi64=@inference_output.32.bin +--expected_output=1x512xi64=@inference_output.33.bin +--expected_output=1x512xi64=@inference_output.34.bin +--expected_output=1x512xi64=@inference_output.35.bin +--expected_output=1x512xi64=@inference_output.36.bin +--expected_output=1x512xi64=@inference_output.37.bin +--expected_output=1x512xi64=@inference_output.38.bin +--expected_output=1x512xi64=@inference_output.39.bin +--expected_output=1x512xi64=@inference_output.40.bin +--expected_output=1x512xi64=@inference_output.41.bin +--expected_output=1x512xi64=@inference_output.42.bin +--expected_output=1x512xi64=@inference_output.43.bin +--expected_output=1x512xi64=@inference_output.44.bin +--expected_output=1x512xi64=@inference_output.45.bin +--expected_output=1x512xi64=@inference_output.46.bin +--expected_output=1x512xi64=@inference_output.47.bin +--expected_output=1x512xi64=@inference_output.48.bin +--expected_output=1x512xi64=@inference_output.49.bin +--expected_output=1x512xi64=@inference_output.50.bin +--expected_output=1x512xi64=@inference_output.51.bin +--expected_output=1x512xi64=@inference_output.52.bin +--expected_output=1x512xi64=@inference_output.53.bin +--expected_output=1x512xi64=@inference_output.54.bin +--expected_output=1x512xi64=@inference_output.55.bin +--expected_output=1x512xi64=@inference_output.56.bin +--expected_output=1x512xi64=@inference_output.57.bin +--expected_output=1x512xi64=@inference_output.58.bin +--expected_output=1x512xi64=@inference_output.59.bin +--expected_output=1x512xi64=@inference_output.60.bin +--expected_output=1x512xi64=@inference_output.61.bin +--expected_output=1x512xi64=@inference_output.62.bin +--expected_output=1x512xi64=@inference_output.63.bin +--expected_output=1x512xi64=@inference_output.64.bin +--expected_output=1x512xi64=@inference_output.65.bin +--expected_output=1x512xi64=@inference_output.66.bin +--expected_output=1x512xi64=@inference_output.67.bin +--expected_output=1x512xi64=@inference_output.68.bin +--expected_output=1x512xi64=@inference_output.69.bin +--expected_output=1x512xi64=@inference_output.70.bin +--expected_output=1x512xi64=@inference_output.71.bin +--expected_output=1x512xi64=@inference_output.72.bin +--expected_output=1x512xi64=@inference_output.73.bin +--expected_output=1x512xi64=@inference_output.74.bin +--expected_output=1x512xi64=@inference_output.75.bin +--expected_output=1x512xi64=@inference_output.76.bin +--expected_output=1x512xi64=@inference_output.77.bin +--expected_output=1x512xi64=@inference_output.78.bin +--expected_output=1x512xi64=@inference_output.79.bin +--expected_output=1x512xi64=@inference_output.80.bin +--expected_output=1x512xi64=@inference_output.81.bin +--expected_output=1x512xi64=@inference_output.82.bin +--expected_output=1x512xi64=@inference_output.83.bin +--expected_output=1x512xi64=@inference_output.84.bin +--expected_output=1x512xi64=@inference_output.85.bin +--expected_output=1x512xi64=@inference_output.86.bin +--expected_output=1x512xi64=@inference_output.87.bin +--expected_output=1x512xi64=@inference_output.88.bin +--expected_output=1x512xi64=@inference_output.89.bin +--expected_output=1x512xi64=@inference_output.90.bin +--expected_output=1x512xi64=@inference_output.91.bin +--expected_output=1x512xi64=@inference_output.92.bin +--expected_output=1x512xi64=@inference_output.93.bin +--expected_output=1x512xi64=@inference_output.94.bin +--expected_output=1x512xi64=@inference_output.95.bin +--expected_output=1x512xi64=@inference_output.96.bin +--expected_output=1x512xi64=@inference_output.97.bin diff --git a/iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json b/iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json new file mode 100644 index 000000000..6c3acb325 --- /dev/null +++ b/iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json @@ -0,0 +1,112 @@ +{ + "file_format": "test_cases_v0", + "test_cases": [ + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_files": [ + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_input.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_input.1.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_input.2.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.1.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.2.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.3.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.4.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.5.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.6.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.7.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.8.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.9.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.10.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.11.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.12.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.13.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.14.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.15.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.16.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.17.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.18.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.19.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.20.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.21.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.22.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.23.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.24.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.25.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.26.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.27.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.28.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.29.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.30.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.31.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.32.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.33.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.34.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.35.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.36.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.37.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.38.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.39.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.40.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.41.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.42.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.43.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.44.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.45.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.46.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.47.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.48.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.49.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.50.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.51.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.52.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.53.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.54.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.55.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.56.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.57.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.58.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.59.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.60.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.61.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.62.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.63.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.64.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.65.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.66.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.67.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.68.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.69.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.70.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.71.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.72.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.73.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.74.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.75.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.76.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.77.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.78.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.79.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.80.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.81.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.82.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.83.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.84.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.85.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.86.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.87.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.88.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.89.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.90.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.91.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.92.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.93.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.94.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.95.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.96.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.97.bin" + ] + } + ] +} From 9b9f7b18beeb6145d1605bde43f5d91dcbda9b2a Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 03:44:05 -0700 Subject: [PATCH 02/11] comment out jobs that we don't care about for validation purposes now --- .github/workflows/test_iree.yml | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml index bfd0f95bb..339fdb5fa 100644 --- a/.github/workflows/test_iree.yml +++ b/.github/workflows/test_iree.yml @@ -166,50 +166,50 @@ jobs: --durations=0 \ --config-files=${MODELS_CONFIG_FILE_PATH} - - name: "Running SDXL special model tests" - id: special_models_sdxl - if: ${{ !cancelled() }} - run: | - source ${VENV_DIR}/bin/activate - pytest iree_special_models/sdxl \ - -k ${{ matrix.backend }} \ - -rpfE \ - --capture=no \ - --log-cli-level=info \ - --timeout=1200 \ - --durations=0 + # - name: "Running SDXL special model tests" + # id: special_models_sdxl + # if: ${{ !cancelled() }} + # run: | + # source ${VENV_DIR}/bin/activate + # pytest iree_special_models/sdxl \ + # -k ${{ matrix.backend }} \ + # -rpfE \ + # --capture=no \ + # --log-cli-level=info \ + # --timeout=1200 \ + # --durations=0 - - name: "Running SD3 special model tests" - id: special_models_sd3 - if: ${{ !cancelled() }} - run: | - source ${VENV_DIR}/bin/activate - pytest iree_special_models/sd3 \ - -k ${{ matrix.backend }} \ - -rpfE \ - --capture=no \ - --log-cli-level=info \ - --timeout=1200 \ - --durations=0 + # - name: "Running SD3 special model tests" + # id: special_models_sd3 + # if: ${{ !cancelled() }} + # run: | + # source ${VENV_DIR}/bin/activate + # pytest iree_special_models/sd3 \ + # -k ${{ matrix.backend }} \ + # -rpfE \ + # --capture=no \ + # --log-cli-level=info \ + # --timeout=1200 \ + # --durations=0 - - name: "Running SDXL ROCm pipeline benchmark" - if: | - contains(matrix.name, 'rocm') && - steps.special_models_sdxl.outcome == 'success' - run: | - source ${VENV_DIR}/bin/activate - pytest iree_tests/benchmarks/sdxl/benchmark_sdxl_rocm.py \ - --goldentime-rocm-e2e-ms 1661.5 \ - --goldentime-rocm-unet-ms 450.5 \ - --goldentime-rocm-clip-ms 19 \ - --goldentime-rocm-vae-ms 295 \ - --goldendispatch-rocm-unet 1714 \ - --goldendispatch-rocm-clip 1569 \ - --goldendispatch-rocm-vae 248 \ - --goldensize-rocm-unet-bytes 2088217 \ - --goldensize-rocm-clip-bytes 785493 \ - --goldensize-rocm-vae-bytes 762067 \ - --gpu-number 6 \ - --rocm-chip gfx90a \ - --log-cli-level=info - echo "$(> $GITHUB_STEP_SUMMARY + # - name: "Running SDXL ROCm pipeline benchmark" + # if: | + # contains(matrix.name, 'rocm') && + # steps.special_models_sdxl.outcome == 'success' + # run: | + # source ${VENV_DIR}/bin/activate + # pytest iree_tests/benchmarks/sdxl/benchmark_sdxl_rocm.py \ + # --goldentime-rocm-e2e-ms 1661.5 \ + # --goldentime-rocm-unet-ms 450.5 \ + # --goldentime-rocm-clip-ms 19 \ + # --goldentime-rocm-vae-ms 295 \ + # --goldendispatch-rocm-unet 1714 \ + # --goldendispatch-rocm-clip 1569 \ + # --goldendispatch-rocm-vae 248 \ + # --goldensize-rocm-unet-bytes 2088217 \ + # --goldensize-rocm-clip-bytes 785493 \ + # --goldensize-rocm-vae-bytes 762067 \ + # --gpu-number 6 \ + # --rocm-chip gfx90a \ + # --log-cli-level=info + # echo "$(> $GITHUB_STEP_SUMMARY From 10d44e74e223bfdaf0fb75ba2e2f7fdb3f4e72f4 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 04:00:00 -0700 Subject: [PATCH 03/11] try with added flag --- iree_tests/configs/models_cpu_llvm_task.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iree_tests/configs/models_cpu_llvm_task.json b/iree_tests/configs/models_cpu_llvm_task.json index a2c00ec04..ee2a6f520 100644 --- a/iree_tests/configs/models_cpu_llvm_task.json +++ b/iree_tests/configs/models_cpu_llvm_task.json @@ -2,7 +2,8 @@ "config_name": "cpu_llvm_task", "iree_compile_flags" : [ "--iree-hal-target-backends=llvm-cpu", - "--iree-llvmcpu-target-cpu-features=host" + "--iree-llvmcpu-target-cpu-features=host", + "--iree-input-demote-i64-to-i32" ], "iree_run_module_flags": [ "--device=local-task" From a5c0f77fb775997b5c1ca8c3d67998d7fa00d860 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 04:23:25 -0700 Subject: [PATCH 04/11] update mobilebert shape output --- .../onnx-export/mobilebert-uncased/real_weights_data_flags.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt index 41ba48aa1..1d7ffa1b5 100644 --- a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt @@ -1,2 +1,2 @@ --input=1x128xi64=@inference_input.0.bin ---expected_output=1x128xi64=@inference_output.0.bin +--expected_output=1x2xi64=@inference_output.0.bin From a08861413f318fc2d14929b9c92cb1c470fc8282 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 04:26:11 -0700 Subject: [PATCH 05/11] remove all expected fails to see if works as intended --- iree_tests/configs/models_cpu_llvm_task.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iree_tests/configs/models_cpu_llvm_task.json b/iree_tests/configs/models_cpu_llvm_task.json index ee2a6f520..5141d403e 100644 --- a/iree_tests/configs/models_cpu_llvm_task.json +++ b/iree_tests/configs/models_cpu_llvm_task.json @@ -15,9 +15,9 @@ "pytorch/models/direct/resnet50", // TODO(#17344): need to regenerate .mlirbc ], "expected_run_failures": [ - "pytorch/models/onnx-export/mit-b0", - "pytorch/models/direct/mobilebert-uncased", - "pytorch/models/direct/t5-base", - "pytorch/models/direct/t5-large" + // "pytorch/models/onnx-export/mit-b0", + // "pytorch/models/direct/mobilebert-uncased", + // "pytorch/models/direct/t5-base", + // "pytorch/models/direct/t5-large" ] } From a0d2d529283361ceb7cc6dd959ac69fb7cfacf8f Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 04:34:44 -0700 Subject: [PATCH 06/11] fix all shapes --- .../mit-b0/real_weights_data_flags.txt | 2 +- .../t5-base/real_weights_data_flags.txt | 100 ++++----- .../t5-large/real_weights_data_flags.txt | 196 +++++++++--------- 3 files changed, 149 insertions(+), 149 deletions(-) diff --git a/iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt index 56855264e..7118e494d 100644 --- a/iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/mit-b0/real_weights_data_flags.txt @@ -1,3 +1,3 @@ --input=1x3x512x512xf32=@inference_input.0.bin ---expected_output=1x3x512x512xf32=@inference_output.0.bin +--expected_output=1x1000xf32=@inference_output.0.bin --expected_f32_threshold=0.01f diff --git a/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt index b089ce101..835a1bb86 100644 --- a/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt @@ -1,53 +1,53 @@ --input=1x512xi64=@inference_input.0.bin --input=1x512xf32=@inference_input.1.bin --input=1x512xi64=@inference_input.2.bin ---expected_output=1x512xi64=@inference_output.0.bin ---expected_output=1x512xi64=@inference_output.1.bin ---expected_output=1x512xi64=@inference_output.2.bin ---expected_output=1x512xi64=@inference_output.3.bin ---expected_output=1x512xi64=@inference_output.4.bin ---expected_output=1x512xi64=@inference_output.5.bin ---expected_output=1x512xi64=@inference_output.6.bin ---expected_output=1x512xi64=@inference_output.7.bin ---expected_output=1x512xi64=@inference_output.8.bin ---expected_output=1x512xi64=@inference_output.9.bin ---expected_output=1x512xi64=@inference_output.10.bin ---expected_output=1x512xi64=@inference_output.11.bin ---expected_output=1x512xi64=@inference_output.12.bin ---expected_output=1x512xi64=@inference_output.13.bin ---expected_output=1x512xi64=@inference_output.14.bin ---expected_output=1x512xi64=@inference_output.15.bin ---expected_output=1x512xi64=@inference_output.16.bin ---expected_output=1x512xi64=@inference_output.17.bin ---expected_output=1x512xi64=@inference_output.18.bin ---expected_output=1x512xi64=@inference_output.19.bin ---expected_output=1x512xi64=@inference_output.20.bin ---expected_output=1x512xi64=@inference_output.21.bin ---expected_output=1x512xi64=@inference_output.22.bin ---expected_output=1x512xi64=@inference_output.23.bin ---expected_output=1x512xi64=@inference_output.24.bin ---expected_output=1x512xi64=@inference_output.25.bin ---expected_output=1x512xi64=@inference_output.26.bin ---expected_output=1x512xi64=@inference_output.27.bin ---expected_output=1x512xi64=@inference_output.28.bin ---expected_output=1x512xi64=@inference_output.29.bin ---expected_output=1x512xi64=@inference_output.30.bin ---expected_output=1x512xi64=@inference_output.31.bin ---expected_output=1x512xi64=@inference_output.32.bin ---expected_output=1x512xi64=@inference_output.33.bin ---expected_output=1x512xi64=@inference_output.34.bin ---expected_output=1x512xi64=@inference_output.35.bin ---expected_output=1x512xi64=@inference_output.36.bin ---expected_output=1x512xi64=@inference_output.37.bin ---expected_output=1x512xi64=@inference_output.38.bin ---expected_output=1x512xi64=@inference_output.39.bin ---expected_output=1x512xi64=@inference_output.40.bin ---expected_output=1x512xi64=@inference_output.41.bin ---expected_output=1x512xi64=@inference_output.42.bin ---expected_output=1x512xi64=@inference_output.43.bin ---expected_output=1x512xi64=@inference_output.44.bin ---expected_output=1x512xi64=@inference_output.45.bin ---expected_output=1x512xi64=@inference_output.46.bin ---expected_output=1x512xi64=@inference_output.47.bin ---expected_output=1x512xi64=@inference_output.48.bin ---expected_output=1x512xi64=@inference_output.49.bin +--expected_output=1x512x768xf32=@inference_output.0.bin +--expected_output=1x12x512x64xf32=@inference_output.1.bin +--expected_output=1x12x512x64xf32=@inference_output.2.bin +--expected_output=1x12x512x64xf32=@inference_output.3.bin +--expected_output=1x12x512x64xf32=@inference_output.4.bin +--expected_output=1x12x512x64xf32=@inference_output.5.bin +--expected_output=1x12x512x64xf32=@inference_output.6.bin +--expected_output=1x12x512x64xf32=@inference_output.7.bin +--expected_output=1x12x512x64xf32=@inference_output.8.bin +--expected_output=1x12x512x64xf32=@inference_output.9.bin +--expected_output=1x12x512x64xf32=@inference_output.10.bin +--expected_output=1x12x512x64xf32=@inference_output.11.bin +--expected_output=1x12x512x64xf32=@inference_output.12.bin +--expected_output=1x12x512x64xf32=@inference_output.13.bin +--expected_output=1x12x512x64xf32=@inference_output.14.bin +--expected_output=1x12x512x64xf32=@inference_output.15.bin +--expected_output=1x12x512x64xf32=@inference_output.16.bin +--expected_output=1x12x512x64xf32=@inference_output.17.bin +--expected_output=1x12x512x64xf32=@inference_output.18.bin +--expected_output=1x12x512x64xf32=@inference_output.19.bin +--expected_output=1x12x512x64xf32=@inference_output.20.bin +--expected_output=1x12x512x64xf32=@inference_output.21.bin +--expected_output=1x12x512x64xf32=@inference_output.22.bin +--expected_output=1x12x512x64xf32=@inference_output.23.bin +--expected_output=1x12x512x64xf32=@inference_output.24.bin +--expected_output=1x12x512x64xf32=@inference_output.25.bin +--expected_output=1x12x512x64xf32=@inference_output.26.bin +--expected_output=1x12x512x64xf32=@inference_output.27.bin +--expected_output=1x12x512x64xf32=@inference_output.28.bin +--expected_output=1x12x512x64xf32=@inference_output.29.bin +--expected_output=1x12x512x64xf32=@inference_output.30.bin +--expected_output=1x12x512x64xf32=@inference_output.31.bin +--expected_output=1x12x512x64xf32=@inference_output.32.bin +--expected_output=1x12x512x64xf32=@inference_output.33.bin +--expected_output=1x12x512x64xf32=@inference_output.34.bin +--expected_output=1x12x512x64xf32=@inference_output.35.bin +--expected_output=1x12x512x64xf32=@inference_output.36.bin +--expected_output=1x12x512x64xf32=@inference_output.37.bin +--expected_output=1x12x512x64xf32=@inference_output.38.bin +--expected_output=1x12x512x64xf32=@inference_output.39.bin +--expected_output=1x12x512x64xf32=@inference_output.40.bin +--expected_output=1x12x512x64xf32=@inference_output.41.bin +--expected_output=1x12x512x64xf32=@inference_output.42.bin +--expected_output=1x12x512x64xf32=@inference_output.43.bin +--expected_output=1x12x512x64xf32=@inference_output.44.bin +--expected_output=1x12x512x64xf32=@inference_output.45.bin +--expected_output=1x12x512x64xf32=@inference_output.46.bin +--expected_output=1x12x512x64xf32=@inference_output.47.bin +--expected_output=1x12x512x64xf32=@inference_output.48.bin +--expected_output=1x512x768xf32=@inference_output.49.bin diff --git a/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt index 52324c6b0..88eb27d02 100644 --- a/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt @@ -1,101 +1,101 @@ --input=1x512xi64=@inference_input.0.bin --input=1x512xf32=@inference_input.1.bin --input=1x512xi64=@inference_input.2.bin ---expected_output=1x512xi64=@inference_output.0.bin ---expected_output=1x512xi64=@inference_output.1.bin ---expected_output=1x512xi64=@inference_output.2.bin ---expected_output=1x512xi64=@inference_output.3.bin ---expected_output=1x512xi64=@inference_output.4.bin ---expected_output=1x512xi64=@inference_output.5.bin ---expected_output=1x512xi64=@inference_output.6.bin ---expected_output=1x512xi64=@inference_output.7.bin ---expected_output=1x512xi64=@inference_output.8.bin ---expected_output=1x512xi64=@inference_output.9.bin ---expected_output=1x512xi64=@inference_output.10.bin ---expected_output=1x512xi64=@inference_output.11.bin ---expected_output=1x512xi64=@inference_output.12.bin ---expected_output=1x512xi64=@inference_output.13.bin ---expected_output=1x512xi64=@inference_output.14.bin ---expected_output=1x512xi64=@inference_output.15.bin ---expected_output=1x512xi64=@inference_output.16.bin ---expected_output=1x512xi64=@inference_output.17.bin ---expected_output=1x512xi64=@inference_output.18.bin ---expected_output=1x512xi64=@inference_output.19.bin ---expected_output=1x512xi64=@inference_output.20.bin ---expected_output=1x512xi64=@inference_output.21.bin ---expected_output=1x512xi64=@inference_output.22.bin ---expected_output=1x512xi64=@inference_output.23.bin ---expected_output=1x512xi64=@inference_output.24.bin ---expected_output=1x512xi64=@inference_output.25.bin ---expected_output=1x512xi64=@inference_output.26.bin ---expected_output=1x512xi64=@inference_output.27.bin ---expected_output=1x512xi64=@inference_output.28.bin ---expected_output=1x512xi64=@inference_output.29.bin ---expected_output=1x512xi64=@inference_output.30.bin ---expected_output=1x512xi64=@inference_output.31.bin ---expected_output=1x512xi64=@inference_output.32.bin ---expected_output=1x512xi64=@inference_output.33.bin ---expected_output=1x512xi64=@inference_output.34.bin ---expected_output=1x512xi64=@inference_output.35.bin ---expected_output=1x512xi64=@inference_output.36.bin ---expected_output=1x512xi64=@inference_output.37.bin ---expected_output=1x512xi64=@inference_output.38.bin ---expected_output=1x512xi64=@inference_output.39.bin ---expected_output=1x512xi64=@inference_output.40.bin ---expected_output=1x512xi64=@inference_output.41.bin ---expected_output=1x512xi64=@inference_output.42.bin ---expected_output=1x512xi64=@inference_output.43.bin ---expected_output=1x512xi64=@inference_output.44.bin ---expected_output=1x512xi64=@inference_output.45.bin ---expected_output=1x512xi64=@inference_output.46.bin ---expected_output=1x512xi64=@inference_output.47.bin ---expected_output=1x512xi64=@inference_output.48.bin ---expected_output=1x512xi64=@inference_output.49.bin ---expected_output=1x512xi64=@inference_output.50.bin ---expected_output=1x512xi64=@inference_output.51.bin ---expected_output=1x512xi64=@inference_output.52.bin ---expected_output=1x512xi64=@inference_output.53.bin ---expected_output=1x512xi64=@inference_output.54.bin ---expected_output=1x512xi64=@inference_output.55.bin ---expected_output=1x512xi64=@inference_output.56.bin ---expected_output=1x512xi64=@inference_output.57.bin ---expected_output=1x512xi64=@inference_output.58.bin ---expected_output=1x512xi64=@inference_output.59.bin ---expected_output=1x512xi64=@inference_output.60.bin ---expected_output=1x512xi64=@inference_output.61.bin ---expected_output=1x512xi64=@inference_output.62.bin ---expected_output=1x512xi64=@inference_output.63.bin ---expected_output=1x512xi64=@inference_output.64.bin ---expected_output=1x512xi64=@inference_output.65.bin ---expected_output=1x512xi64=@inference_output.66.bin ---expected_output=1x512xi64=@inference_output.67.bin ---expected_output=1x512xi64=@inference_output.68.bin ---expected_output=1x512xi64=@inference_output.69.bin ---expected_output=1x512xi64=@inference_output.70.bin ---expected_output=1x512xi64=@inference_output.71.bin ---expected_output=1x512xi64=@inference_output.72.bin ---expected_output=1x512xi64=@inference_output.73.bin ---expected_output=1x512xi64=@inference_output.74.bin ---expected_output=1x512xi64=@inference_output.75.bin ---expected_output=1x512xi64=@inference_output.76.bin ---expected_output=1x512xi64=@inference_output.77.bin ---expected_output=1x512xi64=@inference_output.78.bin ---expected_output=1x512xi64=@inference_output.79.bin ---expected_output=1x512xi64=@inference_output.80.bin ---expected_output=1x512xi64=@inference_output.81.bin ---expected_output=1x512xi64=@inference_output.82.bin ---expected_output=1x512xi64=@inference_output.83.bin ---expected_output=1x512xi64=@inference_output.84.bin ---expected_output=1x512xi64=@inference_output.85.bin ---expected_output=1x512xi64=@inference_output.86.bin ---expected_output=1x512xi64=@inference_output.87.bin ---expected_output=1x512xi64=@inference_output.88.bin ---expected_output=1x512xi64=@inference_output.89.bin ---expected_output=1x512xi64=@inference_output.90.bin ---expected_output=1x512xi64=@inference_output.91.bin ---expected_output=1x512xi64=@inference_output.92.bin ---expected_output=1x512xi64=@inference_output.93.bin ---expected_output=1x512xi64=@inference_output.94.bin ---expected_output=1x512xi64=@inference_output.95.bin ---expected_output=1x512xi64=@inference_output.96.bin ---expected_output=1x512xi64=@inference_output.97.bin +--expected_output=1x512x1024xf32=@inference_output.0.bin +--expected_output=1x16x512x64xf32=@inference_output.1.bin +--expected_output=1x16x512x64xf32=@inference_output.2.bin +--expected_output=1x16x512x64xf32=@inference_output.3.bin +--expected_output=1x16x512x64xf32=@inference_output.4.bin +--expected_output=1x16x512x64xf32=@inference_output.5.bin +--expected_output=1x16x512x64xf32=@inference_output.6.bin +--expected_output=1x16x512x64xf32=@inference_output.7.bin +--expected_output=1x16x512x64xf32=@inference_output.8.bin +--expected_output=1x16x512x64xf32=@inference_output.9.bin +--expected_output=1x16x512x64xf32=@inference_output.10.bin +--expected_output=1x16x512x64xf32=@inference_output.11.bin +--expected_output=1x16x512x64xf32=@inference_output.12.bin +--expected_output=1x16x512x64xf32=@inference_output.13.bin +--expected_output=1x16x512x64xf32=@inference_output.14.bin +--expected_output=1x16x512x64xf32=@inference_output.15.bin +--expected_output=1x16x512x64xf32=@inference_output.16.bin +--expected_output=1x16x512x64xf32=@inference_output.17.bin +--expected_output=1x16x512x64xf32=@inference_output.18.bin +--expected_output=1x16x512x64xf32=@inference_output.19.bin +--expected_output=1x16x512x64xf32=@inference_output.20.bin +--expected_output=1x16x512x64xf32=@inference_output.21.bin +--expected_output=1x16x512x64xf32=@inference_output.22.bin +--expected_output=1x16x512x64xf32=@inference_output.23.bin +--expected_output=1x16x512x64xf32=@inference_output.24.bin +--expected_output=1x16x512x64xf32=@inference_output.25.bin +--expected_output=1x16x512x64xf32=@inference_output.26.bin +--expected_output=1x16x512x64xf32=@inference_output.27.bin +--expected_output=1x16x512x64xf32=@inference_output.28.bin +--expected_output=1x16x512x64xf32=@inference_output.29.bin +--expected_output=1x16x512x64xf32=@inference_output.30.bin +--expected_output=1x16x512x64xf32=@inference_output.31.bin +--expected_output=1x16x512x64xf32=@inference_output.32.bin +--expected_output=1x16x512x64xf32=@inference_output.33.bin +--expected_output=1x16x512x64xf32=@inference_output.34.bin +--expected_output=1x16x512x64xf32=@inference_output.35.bin +--expected_output=1x16x512x64xf32=@inference_output.36.bin +--expected_output=1x16x512x64xf32=@inference_output.37.bin +--expected_output=1x16x512x64xf32=@inference_output.38.bin +--expected_output=1x16x512x64xf32=@inference_output.39.bin +--expected_output=1x16x512x64xf32=@inference_output.40.bin +--expected_output=1x16x512x64xf32=@inference_output.41.bin +--expected_output=1x16x512x64xf32=@inference_output.42.bin +--expected_output=1x16x512x64xf32=@inference_output.43.bin +--expected_output=1x16x512x64xf32=@inference_output.44.bin +--expected_output=1x16x512x64xf32=@inference_output.45.bin +--expected_output=1x16x512x64xf32=@inference_output.46.bin +--expected_output=1x16x512x64xf32=@inference_output.47.bin +--expected_output=1x16x512x64xf32=@inference_output.48.bin +--expected_output=1x16x512x64xf32=@inference_output.49.bin +--expected_output=1x16x512x64xf32=@inference_output.50.bin +--expected_output=1x16x512x64xf32=@inference_output.51.bin +--expected_output=1x16x512x64xf32=@inference_output.52.bin +--expected_output=1x16x512x64xf32=@inference_output.53.bin +--expected_output=1x16x512x64xf32=@inference_output.54.bin +--expected_output=1x16x512x64xf32=@inference_output.55.bin +--expected_output=1x16x512x64xf32=@inference_output.56.bin +--expected_output=1x16x512x64xf32=@inference_output.57.bin +--expected_output=1x16x512x64xf32=@inference_output.58.bin +--expected_output=1x16x512x64xf32=@inference_output.59.bin +--expected_output=1x16x512x64xf32=@inference_output.60.bin +--expected_output=1x16x512x64xf32=@inference_output.61.bin +--expected_output=1x16x512x64xf32=@inference_output.62.bin +--expected_output=1x16x512x64xf32=@inference_output.63.bin +--expected_output=1x16x512x64xf32=@inference_output.64.bin +--expected_output=1x16x512x64xf32=@inference_output.65.bin +--expected_output=1x16x512x64xf32=@inference_output.66.bin +--expected_output=1x16x512x64xf32=@inference_output.67.bin +--expected_output=1x16x512x64xf32=@inference_output.68.bin +--expected_output=1x16x512x64xf32=@inference_output.69.bin +--expected_output=1x16x512x64xf32=@inference_output.70.bin +--expected_output=1x16x512x64xf32=@inference_output.71.bin +--expected_output=1x16x512x64xf32=@inference_output.72.bin +--expected_output=1x16x512x64xf32=@inference_output.73.bin +--expected_output=1x16x512x64xf32=@inference_output.74.bin +--expected_output=1x16x512x64xf32=@inference_output.75.bin +--expected_output=1x16x512x64xf32=@inference_output.76.bin +--expected_output=1x16x512x64xf32=@inference_output.77.bin +--expected_output=1x16x512x64xf32=@inference_output.78.bin +--expected_output=1x16x512x64xf32=@inference_output.79.bin +--expected_output=1x16x512x64xf32=@inference_output.80.bin +--expected_output=1x16x512x64xf32=@inference_output.81.bin +--expected_output=1x16x512x64xf32=@inference_output.82.bin +--expected_output=1x16x512x64xf32=@inference_output.83.bin +--expected_output=1x16x512x64xf32=@inference_output.84.bin +--expected_output=1x16x512x64xf32=@inference_output.85.bin +--expected_output=1x16x512x64xf32=@inference_output.86.bin +--expected_output=1x16x512x64xf32=@inference_output.87.bin +--expected_output=1x16x512x64xf32=@inference_output.88.bin +--expected_output=1x16x512x64xf32=@inference_output.89.bin +--expected_output=1x16x512x64xf32=@inference_output.90.bin +--expected_output=1x16x512x64xf32=@inference_output.91.bin +--expected_output=1x16x512x64xf32=@inference_output.92.bin +--expected_output=1x16x512x64xf32=@inference_output.93.bin +--expected_output=1x16x512x64xf32=@inference_output.94.bin +--expected_output=1x16x512x64xf32=@inference_output.95.bin +--expected_output=1x16x512x64xf32=@inference_output.96.bin +--expected_output=1x512x1024xf32=@inference_output.97.bin From 7501f7babcf6678c600c1c880db43b7a1df9742c Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 04:51:49 -0700 Subject: [PATCH 07/11] fix remote download json files --- .../onnx-export/mobilebert-uncased/real_weights_data_flags.txt | 2 +- iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json | 2 +- iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt index 1d7ffa1b5..19d01dab9 100644 --- a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt @@ -1,2 +1,2 @@ --input=1x128xi64=@inference_input.0.bin ---expected_output=1x2xi64=@inference_output.0.bin +--expected_output=1x2xf32=@inference_output.0.bin diff --git a/iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json b/iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json index 4f371f843..6cb3f5cee 100644 --- a/iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json +++ b/iree_tests/pytorch/models/onnx-export/t5-base/test_cases.json @@ -58,7 +58,7 @@ "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.47.bin", "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.48.bin", "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/inference_output.49.bin", - "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/mobilebert-uncased/model.mlirbc" + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-base/model.mlirbc" ] } ] diff --git a/iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json b/iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json index 6c3acb325..21c95bb0c 100644 --- a/iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json +++ b/iree_tests/pytorch/models/onnx-export/t5-large/test_cases.json @@ -105,7 +105,8 @@ "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.94.bin", "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.95.bin", "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.96.bin", - "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.97.bin" + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/inference_output.97.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/shark-test-suite/iree-tests/pytorch/models/onnx-export/t5-large/model.mlirbc" ] } ] From 8411a4a750394f5b1a28184f82072a21f2522118 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 05:04:07 -0700 Subject: [PATCH 08/11] update tolerances and xfails --- iree_tests/configs/models_cpu_llvm_task.json | 3 +-- .../onnx-export/mobilebert-uncased/real_weights_data_flags.txt | 1 + .../models/onnx-export/t5-base/real_weights_data_flags.txt | 1 + .../models/onnx-export/t5-large/real_weights_data_flags.txt | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/iree_tests/configs/models_cpu_llvm_task.json b/iree_tests/configs/models_cpu_llvm_task.json index 5141d403e..1dbdf2818 100644 --- a/iree_tests/configs/models_cpu_llvm_task.json +++ b/iree_tests/configs/models_cpu_llvm_task.json @@ -2,8 +2,7 @@ "config_name": "cpu_llvm_task", "iree_compile_flags" : [ "--iree-hal-target-backends=llvm-cpu", - "--iree-llvmcpu-target-cpu-features=host", - "--iree-input-demote-i64-to-i32" + "--iree-llvmcpu-target-cpu-features=host" ], "iree_run_module_flags": [ "--device=local-task" diff --git a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt index 19d01dab9..d891cfe79 100644 --- a/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/mobilebert-uncased/real_weights_data_flags.txt @@ -1,2 +1,3 @@ --input=1x128xi64=@inference_input.0.bin --expected_output=1x2xf32=@inference_output.0.bin +--expected_f32_threshold=0.01f diff --git a/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt index 835a1bb86..1336184cd 100644 --- a/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/t5-base/real_weights_data_flags.txt @@ -51,3 +51,4 @@ --expected_output=1x12x512x64xf32=@inference_output.47.bin --expected_output=1x12x512x64xf32=@inference_output.48.bin --expected_output=1x512x768xf32=@inference_output.49.bin +--expected_f32_threshold=0.01f diff --git a/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt b/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt index 88eb27d02..ec0dacc14 100644 --- a/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/onnx-export/t5-large/real_weights_data_flags.txt @@ -99,3 +99,4 @@ --expected_output=1x16x512x64xf32=@inference_output.95.bin --expected_output=1x16x512x64xf32=@inference_output.96.bin --expected_output=1x512x1024xf32=@inference_output.97.bin +--expected_f32_threshold=0.01f From bb90c561d4f920a53bb58b4f14895bb73928cc62 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 05:05:10 -0700 Subject: [PATCH 09/11] xfails --- iree_tests/configs/models_cpu_llvm_task.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/iree_tests/configs/models_cpu_llvm_task.json b/iree_tests/configs/models_cpu_llvm_task.json index 1dbdf2818..59369926b 100644 --- a/iree_tests/configs/models_cpu_llvm_task.json +++ b/iree_tests/configs/models_cpu_llvm_task.json @@ -2,7 +2,8 @@ "config_name": "cpu_llvm_task", "iree_compile_flags" : [ "--iree-hal-target-backends=llvm-cpu", - "--iree-llvmcpu-target-cpu-features=host" + "--iree-llvmcpu-target-cpu-features=host", + "--iree-input-demote-i64-to-i32" ], "iree_run_module_flags": [ "--device=local-task" @@ -14,9 +15,8 @@ "pytorch/models/direct/resnet50", // TODO(#17344): need to regenerate .mlirbc ], "expected_run_failures": [ - // "pytorch/models/onnx-export/mit-b0", - // "pytorch/models/direct/mobilebert-uncased", - // "pytorch/models/direct/t5-base", - // "pytorch/models/direct/t5-large" + "pytorch/models/direct/mobilebert-uncased", + "pytorch/models/direct/t5-base", + "pytorch/models/direct/t5-large" ] } From 3d9bd2c122995c0bb2137990ba921b696bc70c10 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 06:23:30 -0700 Subject: [PATCH 10/11] update xfail --- iree_tests/configs/models_cpu_llvm_task.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/iree_tests/configs/models_cpu_llvm_task.json b/iree_tests/configs/models_cpu_llvm_task.json index 59369926b..d11522020 100644 --- a/iree_tests/configs/models_cpu_llvm_task.json +++ b/iree_tests/configs/models_cpu_llvm_task.json @@ -15,8 +15,6 @@ "pytorch/models/direct/resnet50", // TODO(#17344): need to regenerate .mlirbc ], "expected_run_failures": [ - "pytorch/models/direct/mobilebert-uncased", - "pytorch/models/direct/t5-base", - "pytorch/models/direct/t5-large" + "pytorch/models/onnx-export/mobilebert-uncased", ] } From 34aff57c3791504f32bc691f47170c40ac95c493 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 3 Jul 2024 06:39:58 -0700 Subject: [PATCH 11/11] uncomment out tests now that verified --- .github/workflows/test_iree.yml | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml index 339fdb5fa..bfd0f95bb 100644 --- a/.github/workflows/test_iree.yml +++ b/.github/workflows/test_iree.yml @@ -166,50 +166,50 @@ jobs: --durations=0 \ --config-files=${MODELS_CONFIG_FILE_PATH} - # - name: "Running SDXL special model tests" - # id: special_models_sdxl - # if: ${{ !cancelled() }} - # run: | - # source ${VENV_DIR}/bin/activate - # pytest iree_special_models/sdxl \ - # -k ${{ matrix.backend }} \ - # -rpfE \ - # --capture=no \ - # --log-cli-level=info \ - # --timeout=1200 \ - # --durations=0 + - name: "Running SDXL special model tests" + id: special_models_sdxl + if: ${{ !cancelled() }} + run: | + source ${VENV_DIR}/bin/activate + pytest iree_special_models/sdxl \ + -k ${{ matrix.backend }} \ + -rpfE \ + --capture=no \ + --log-cli-level=info \ + --timeout=1200 \ + --durations=0 - # - name: "Running SD3 special model tests" - # id: special_models_sd3 - # if: ${{ !cancelled() }} - # run: | - # source ${VENV_DIR}/bin/activate - # pytest iree_special_models/sd3 \ - # -k ${{ matrix.backend }} \ - # -rpfE \ - # --capture=no \ - # --log-cli-level=info \ - # --timeout=1200 \ - # --durations=0 + - name: "Running SD3 special model tests" + id: special_models_sd3 + if: ${{ !cancelled() }} + run: | + source ${VENV_DIR}/bin/activate + pytest iree_special_models/sd3 \ + -k ${{ matrix.backend }} \ + -rpfE \ + --capture=no \ + --log-cli-level=info \ + --timeout=1200 \ + --durations=0 - # - name: "Running SDXL ROCm pipeline benchmark" - # if: | - # contains(matrix.name, 'rocm') && - # steps.special_models_sdxl.outcome == 'success' - # run: | - # source ${VENV_DIR}/bin/activate - # pytest iree_tests/benchmarks/sdxl/benchmark_sdxl_rocm.py \ - # --goldentime-rocm-e2e-ms 1661.5 \ - # --goldentime-rocm-unet-ms 450.5 \ - # --goldentime-rocm-clip-ms 19 \ - # --goldentime-rocm-vae-ms 295 \ - # --goldendispatch-rocm-unet 1714 \ - # --goldendispatch-rocm-clip 1569 \ - # --goldendispatch-rocm-vae 248 \ - # --goldensize-rocm-unet-bytes 2088217 \ - # --goldensize-rocm-clip-bytes 785493 \ - # --goldensize-rocm-vae-bytes 762067 \ - # --gpu-number 6 \ - # --rocm-chip gfx90a \ - # --log-cli-level=info - # echo "$(> $GITHUB_STEP_SUMMARY + - name: "Running SDXL ROCm pipeline benchmark" + if: | + contains(matrix.name, 'rocm') && + steps.special_models_sdxl.outcome == 'success' + run: | + source ${VENV_DIR}/bin/activate + pytest iree_tests/benchmarks/sdxl/benchmark_sdxl_rocm.py \ + --goldentime-rocm-e2e-ms 1661.5 \ + --goldentime-rocm-unet-ms 450.5 \ + --goldentime-rocm-clip-ms 19 \ + --goldentime-rocm-vae-ms 295 \ + --goldendispatch-rocm-unet 1714 \ + --goldendispatch-rocm-clip 1569 \ + --goldendispatch-rocm-vae 248 \ + --goldensize-rocm-unet-bytes 2088217 \ + --goldensize-rocm-clip-bytes 785493 \ + --goldensize-rocm-vae-bytes 762067 \ + --gpu-number 6 \ + --rocm-chip gfx90a \ + --log-cli-level=info + echo "$(> $GITHUB_STEP_SUMMARY