From acac09f7d495c12e1b69918f16754b99afa94b53 Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Thu, 27 Jun 2024 21:55:04 +0000 Subject: [PATCH 01/10] regression benchmark --- .github/workflows/regression-benchmark.yml | 59 +++++++++++++++++++ .../python/benchmarks/regression_benchmark.py | 51 ++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 .github/workflows/regression-benchmark.yml create mode 100644 benchmark/python/benchmarks/regression_benchmark.py diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml new file mode 100644 index 0000000..c7a0970 --- /dev/null +++ b/.github/workflows/regression-benchmark.yml @@ -0,0 +1,59 @@ +name: Regression benchmark + +on: + workflow_run: + workflows: [Build and test] + types: [completed] + # branches: [main] + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Performance regression check + runs-on: ubuntu-latest + env: + CACHE_DIR: ${{ github.workspace }}/.ccache + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python Version + uses: actions/setup-python@v5 + with: + python-version: 3.11 # Install the python version needed + + - name: Set PYTHONPATH + run: export PYTHONPATH=build/tools/mpact/python_packages/mpact + shell: bash + + - name: Build + run: cmake --build build --target build-benchmark-mpact + + - name: Install requirements + run: | + export CCACHE_DIR=${{ env.CACHE_DIR }} + python -m pip install --upgrade pip + python -m pip install pytest pytest-benchmark + + - name: Run benchmark + run: pytest benchmark/python/benchmarks/regression_benchmark.py --benchmark-json output.json + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'pytest' + output-file-path: output.json + fail-on-alert: true + # GitHub API token to make a commit comment + github-token: ${{ secrets.GITHUB_TOKEN }} + # Enable alert commit comment + comment-on-alert: true + # Mention @rhysd in the commit comment + alert-comment-cc-users: '@yinying-lisa-li' + # Push and deploy GitHub pages branch automatically + auto-push: true \ No newline at end of file diff --git a/benchmark/python/benchmarks/regression_benchmark.py b/benchmark/python/benchmarks/regression_benchmark.py new file mode 100644 index 0000000..6f5196e --- /dev/null +++ b/benchmark/python/benchmarks/regression_benchmark.py @@ -0,0 +1,51 @@ +import pytest +from mpact.models.kernels import * +from mpact_benchmark.utils.tensor_generator import generate_tensor + +SHAPE = (1024, 1024) +SPARSITY = 0.8 + +dense_tensor1 = generate_tensor(0, SHAPE, SPARSITY) +dense_tensor2 = generate_tensor(1, SHAPE, SPARSITY) +dense_tensor3 = generate_tensor(2, SHAPE, SPARSITY) +dense_vector = generate_tensor(1, (SHAPE[0],), SPARSITY) + +sparse_tensor1 = dense_tensor1.to_sparse_csr() +sparse_tensor2 = dense_tensor2.to_sparse_csr() +sparse_tensor3 = dense_tensor3.to_sparse_csr() + +def test_mv_dense(benchmark): + benchmark(MVNet(), dense_tensor1, dense_vector) + +def test_mm_dense(benchmark): + benchmark(MMNet(), dense_tensor1, dense_tensor2) + +def test_add_dense(benchmark): + benchmark(AddNet(), dense_tensor1, dense_tensor2) + +def test_mul_dense(benchmark): + benchmark(MulNet(), dense_tensor1, dense_tensor2) + +def test_nop_dense(benchmark): + benchmark(SelfNet(), dense_tensor1) + +def test_sddmm_dense(benchmark): + benchmark(SDDMMNet(), dense_tensor1, dense_tensor2, dense_tensor3) + +def test_mv_sparse(benchmark): + benchmark(MVNet(), sparse_tensor1, dense_vector) + +def test_mm_sparse(benchmark): + benchmark(MMNet(), sparse_tensor1, sparse_tensor2) + +def test_add_sparse(benchmark): + benchmark(AddNet(), sparse_tensor1, sparse_tensor2) + +def test_mul_sparse(benchmark): + benchmark(MulNet(), sparse_tensor1, sparse_tensor2) + +def test_nop_sparse(benchmark): + benchmark(SelfNet(), sparse_tensor1) + +def test_sddmm_sparse(benchmark): + benchmark(SDDMMNet(), sparse_tensor1, dense_tensor2, dense_tensor3) \ No newline at end of file From a16609aea32765d11e7342d8e092211840c9a099 Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Thu, 27 Jun 2024 22:04:12 +0000 Subject: [PATCH 02/10] fix format --- .github/workflows/regression-benchmark.yml | 2 +- .../python/benchmarks/regression_benchmark.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml index c7a0970..b2708c4 100644 --- a/.github/workflows/regression-benchmark.yml +++ b/.github/workflows/regression-benchmark.yml @@ -56,4 +56,4 @@ jobs: # Mention @rhysd in the commit comment alert-comment-cc-users: '@yinying-lisa-li' # Push and deploy GitHub pages branch automatically - auto-push: true \ No newline at end of file + auto-push: true diff --git a/benchmark/python/benchmarks/regression_benchmark.py b/benchmark/python/benchmarks/regression_benchmark.py index 6f5196e..6b3d0e4 100644 --- a/benchmark/python/benchmarks/regression_benchmark.py +++ b/benchmark/python/benchmarks/regression_benchmark.py @@ -14,38 +14,50 @@ sparse_tensor2 = dense_tensor2.to_sparse_csr() sparse_tensor3 = dense_tensor3.to_sparse_csr() + def test_mv_dense(benchmark): benchmark(MVNet(), dense_tensor1, dense_vector) + def test_mm_dense(benchmark): benchmark(MMNet(), dense_tensor1, dense_tensor2) + def test_add_dense(benchmark): benchmark(AddNet(), dense_tensor1, dense_tensor2) + def test_mul_dense(benchmark): benchmark(MulNet(), dense_tensor1, dense_tensor2) + def test_nop_dense(benchmark): benchmark(SelfNet(), dense_tensor1) + def test_sddmm_dense(benchmark): benchmark(SDDMMNet(), dense_tensor1, dense_tensor2, dense_tensor3) + def test_mv_sparse(benchmark): benchmark(MVNet(), sparse_tensor1, dense_vector) + def test_mm_sparse(benchmark): benchmark(MMNet(), sparse_tensor1, sparse_tensor2) + def test_add_sparse(benchmark): benchmark(AddNet(), sparse_tensor1, sparse_tensor2) + def test_mul_sparse(benchmark): benchmark(MulNet(), sparse_tensor1, sparse_tensor2) + def test_nop_sparse(benchmark): benchmark(SelfNet(), sparse_tensor1) + def test_sddmm_sparse(benchmark): - benchmark(SDDMMNet(), sparse_tensor1, dense_tensor2, dense_tensor3) \ No newline at end of file + benchmark(SDDMMNet(), sparse_tensor1, dense_tensor2, dense_tensor3) From 400a1dc51e86f363634087bd391425c7aa6759ef Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Thu, 27 Jun 2024 22:17:35 +0000 Subject: [PATCH 03/10] change trigger --- .github/workflows/regression-benchmark.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml index b2708c4..71dd72a 100644 --- a/.github/workflows/regression-benchmark.yml +++ b/.github/workflows/regression-benchmark.yml @@ -1,10 +1,15 @@ name: Regression benchmark on: - workflow_run: - workflows: [Build and test] - types: [completed] - # branches: [main] + # workflow_run: + # workflows: + # - Check code formatting + # types: + # - completed + # branches: + # - regression_bench + pull_request: + branches: [ "main" ] permissions: contents: write From 318251c4cf0dc132bb520a8950cd412eff0a0ac2 Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Thu, 27 Jun 2024 22:22:50 +0000 Subject: [PATCH 04/10] add cmake --- .github/workflows/regression-benchmark.yml | 30 ++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml index 71dd72a..8031623 100644 --- a/.github/workflows/regression-benchmark.yml +++ b/.github/workflows/regression-benchmark.yml @@ -36,14 +36,40 @@ jobs: run: export PYTHONPATH=build/tools/mpact/python_packages/mpact shell: bash - - name: Build - run: cmake --build build --target build-benchmark-mpact + - name: Set up ccache + uses: hendrikmuhs/ccache-action@v1.2 - name: Install requirements run: | export CCACHE_DIR=${{ env.CACHE_DIR }} python -m pip install --upgrade pip + python -m pip install --upgrade pip python -m pip install pytest pytest-benchmark + python -m pip install -r externals/torch-mlir/requirements.txt + python -m pip install -r externals/torch-mlir/torchvision-requirements.txt + + - name: Create build directory + run: mkdir build + + - name: Configure CMake + run: > + cmake -GNinja -Bbuild + -DCMAKE_BUILD_TYPE=Release + -DLLVM_ENABLE_PROJECTS=mlir + -DLLVM_ENABLE_ASSERTIONS=ON + -DLLVM_EXTERNAL_PROJECTS="torch-mlir;mpact" + -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="${PWD}/externals/torch-mlir" + -DLLVM_EXTERNAL_MPACT_SOURCE_DIR="${PWD}" + -DLLVM_TARGETS_TO_BUILD=host + -DMLIR_ENABLE_BINDINGS_PYTHON=ON + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DCMAKE_C_COMPILER=clang + -DCMAKE_CXX_COMPILER=clang++ + "externals/torch-mlir/externals/llvm-project/llvm" + + - name: Build + run: cmake --build build --target build-benchmark-mpact - name: Run benchmark run: pytest benchmark/python/benchmarks/regression_benchmark.py --benchmark-json output.json From 82e0b7c644c2b9440bcff64ed21dd982709f793c Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Thu, 27 Jun 2024 22:43:17 +0000 Subject: [PATCH 05/10] build --- .github/workflows/regression-benchmark.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml index 8031623..524f8c4 100644 --- a/.github/workflows/regression-benchmark.yml +++ b/.github/workflows/regression-benchmark.yml @@ -20,7 +20,8 @@ jobs: name: Performance regression check runs-on: ubuntu-latest env: - CACHE_DIR: ${{ github.workspace }}/.ccache + CACHE_DIR: ${{ github.workspace }}/.ccache + PYTHONPATH: ${{ github.workspace }}/build/tools/mpact/python_packages/mpact steps: - uses: actions/checkout@v4 with: @@ -32,10 +33,6 @@ jobs: with: python-version: 3.11 # Install the python version needed - - name: Set PYTHONPATH - run: export PYTHONPATH=build/tools/mpact/python_packages/mpact - shell: bash - - name: Set up ccache uses: hendrikmuhs/ccache-action@v1.2 From 5dc0fa4ef942eeb866988fbb05f3a20c4728c796 Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Fri, 28 Jun 2024 18:30:46 +0000 Subject: [PATCH 06/10] test comment alert --- .github/workflows/regression-benchmark.yml | 3 ++- benchmark/python/benchmarks/regression_benchmark.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml index 524f8c4..5d2d6ee 100644 --- a/.github/workflows/regression-benchmark.yml +++ b/.github/workflows/regression-benchmark.yml @@ -81,7 +81,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} # Enable alert commit comment comment-on-alert: true - # Mention @rhysd in the commit comment + # Mention @yinying-lisa-li in the commit comment alert-comment-cc-users: '@yinying-lisa-li' # Push and deploy GitHub pages branch automatically auto-push: true + alert-threshold: 150% diff --git a/benchmark/python/benchmarks/regression_benchmark.py b/benchmark/python/benchmarks/regression_benchmark.py index 6b3d0e4..66050c2 100644 --- a/benchmark/python/benchmarks/regression_benchmark.py +++ b/benchmark/python/benchmarks/regression_benchmark.py @@ -10,6 +10,8 @@ dense_tensor3 = generate_tensor(2, SHAPE, SPARSITY) dense_vector = generate_tensor(1, (SHAPE[0],), SPARSITY) +temp = generate_tensor(0, (2046, 2046), 0.5) + sparse_tensor1 = dense_tensor1.to_sparse_csr() sparse_tensor2 = dense_tensor2.to_sparse_csr() sparse_tensor3 = dense_tensor3.to_sparse_csr() @@ -36,7 +38,7 @@ def test_nop_dense(benchmark): def test_sddmm_dense(benchmark): - benchmark(SDDMMNet(), dense_tensor1, dense_tensor2, dense_tensor3) + benchmark(SDDMMNet(), temp, temp, temp) def test_mv_sparse(benchmark): From c8f7db4befa5631607f57392a42c0c3eef095b14 Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Fri, 28 Jun 2024 21:50:29 +0000 Subject: [PATCH 07/10] change threshold --- .github/workflows/regression-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml index 5d2d6ee..865c392 100644 --- a/.github/workflows/regression-benchmark.yml +++ b/.github/workflows/regression-benchmark.yml @@ -85,4 +85,4 @@ jobs: alert-comment-cc-users: '@yinying-lisa-li' # Push and deploy GitHub pages branch automatically auto-push: true - alert-threshold: 150% + alert-threshold: 120% From 4b3c2668ba82622c4923d8c4c9c1baa69c7ddacf Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Fri, 28 Jun 2024 21:55:37 +0000 Subject: [PATCH 08/10] bump --- externals/torch-mlir | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/torch-mlir b/externals/torch-mlir index 6fece25..c7d52f6 160000 --- a/externals/torch-mlir +++ b/externals/torch-mlir @@ -1 +1 @@ -Subproject commit 6fece25ff3203bbc538756beb83fd513c19bcd7d +Subproject commit c7d52f63b482b2c30f4efb435ce0cc2efeab25d9 From 59fc5a073bebb037b2051bbe7c8a63dcf2ad82dc Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Fri, 28 Jun 2024 22:08:30 +0000 Subject: [PATCH 09/10] permission --- .github/workflows/regression-benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml index 865c392..81dda81 100644 --- a/.github/workflows/regression-benchmark.yml +++ b/.github/workflows/regression-benchmark.yml @@ -14,6 +14,8 @@ on: permissions: contents: write deployments: write + pull-requests: write + repository-projects: write jobs: benchmark: From 74f6291a82632c69528c283f0ad04fb7d5d65e63 Mon Sep 17 00:00:00 2001 From: yinying-lisa-li Date: Fri, 28 Jun 2024 22:24:16 +0000 Subject: [PATCH 10/10] test comment --- benchmark/python/benchmarks/regression_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/python/benchmarks/regression_benchmark.py b/benchmark/python/benchmarks/regression_benchmark.py index 66050c2..1f3c215 100644 --- a/benchmark/python/benchmarks/regression_benchmark.py +++ b/benchmark/python/benchmarks/regression_benchmark.py @@ -62,4 +62,4 @@ def test_nop_sparse(benchmark): def test_sddmm_sparse(benchmark): - benchmark(SDDMMNet(), sparse_tensor1, dense_tensor2, dense_tensor3) + benchmark(SDDMMNet(), temp, temp, temp)