Benchmark: matmul ukernel vs direct codegen #1773
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| workflow_call: | |
| pull_request: | |
| merge_group: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| # A PR number if a pull request and otherwise the commit hash. This cancels | |
| # queued and in-progress runs for the same PR (presubmit) or commit | |
| # (postsubmit). | |
| group: ci-build-test-cpp-linux-${{ github.event.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| build_test_linux: | |
| name: Build and Test (Linux, ASSERTIONS) | |
| runs-on: nod-ai-shared-cpubuilder-manylinux-x86_64 | |
| strategy: | |
| fail-fast: true | |
| env: | |
| CACHE_DIR: ${{ github.workspace }}/.container-cache | |
| steps: | |
| - name: Configure local git mirrors | |
| run: | | |
| /gitmirror/scripts/trigger_update_mirrors.sh | |
| /gitmirror/scripts/git_config.sh | |
| - name: "Checking out repository" | |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 | |
| with: | |
| submodules: recursive | |
| - name: Sync source deps | |
| run: | | |
| python ./sync_deps.py | |
| - name: Enable cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ${{ env.CACHE_DIR }} | |
| key: linux-build-test-cpp-asserts-manylinux-v2-${{ github.sha }} | |
| restore-keys: | | |
| linux-build-test-cpp- | |
| - name: Build packages | |
| run: | | |
| export cache_dir="${{ env.CACHE_DIR }}" | |
| bash build_tools/ci/build_test_cpp.sh | |
| - name: Create artifacts | |
| if: ${{ !cancelled() }} | |
| run: | | |
| # TODO: https://github.com/openxla/iree/issues/16230 | |
| # We should not be packaging clang and llvm-link in the default dist | |
| # installs. | |
| rm -f iree-install/bin/clang* | |
| rm -f iree-install/bin/llvm-link* | |
| tar cf iree-dist.tar -C iree-install . -C ../iree-build tools/testing/e2e/iree-e2e-matmul-test | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| name: linux_x86_64_release_packages | |
| path: | | |
| iree-dist.tar | |
| if-no-files-found: warn | |
| - name: Save cache | |
| uses: actions/cache/save@v3 | |
| if: ${{ !cancelled() }} | |
| with: | |
| path: ${{ env.CACHE_DIR }} | |
| key: linux-build-test-cpp-asserts-manylinux-v2-${{ github.sha }} | |
| test_linux: | |
| runs-on: amd7940hs | |
| needs: build_test_linux | |
| strategy: | |
| fail-fast: true | |
| steps: | |
| - name: "Checking out repository" # for test scripts | |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 | |
| with: | |
| submodules: false # not required for testbench | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linux_x86_64_release_packages | |
| - name: Extract artifact | |
| run: | | |
| mkdir iree-install | |
| tar -xf iree-dist.tar -C iree-install | |
| - name: Create venv and install dependencies | |
| run: | | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install https://github.com/Xilinx/mlir-aie/releases/download/latest-wheels/mlir_aie-0.0.1.2024062422+7917990-py3-none-manylinux_2_35_x86_64.whl | |
| pip install -r tests/matmul/requirements.txt | |
| - name: E2E correctness matmul test | |
| run: | | |
| source .venv/bin/activate | |
| # Without this additional line an error like | |
| # | |
| # [XRT] ERROR: Failed to allocate host memory buffer (mmap(len=10616832, prot=3, flags=8193, offset=4294967296) | |
| # failed (err=11): Resource temporarily unavailable), make sure host bank is enabled (see xbutil configure --host-mem) | |
| # iree-amd-aie/runtime/src/iree-amd-aie/driver/xrt/direct_allocator.cc:179: RESOURCE_EXHAUSTED; could not allocate | |
| # memory for buffer; while invoking C++ function matmul_test.generate_random_matrix; while calling import; | |
| # | |
| # might be observed when too much memory is allocated. For example this | |
| # error was seen when running a bf16->f32 matmul with m=n=k=2304. | |
| # | |
| # This line was suggested at https://github.com/Xilinx/mlir-air/issues/566 | |
| # | |
| # Note that this is only half of the fix. It is also necessary that | |
| # the machine that CI is running on has permission to run this line. | |
| # | |
| # This permission can be adding by adding the line | |
| # ``` | |
| # %github ALL=(ALL) NOPASSWD: /usr/bin/prlimit * | |
| # ``` | |
| # | |
| # to the file /etc/sudoers.d/github, which can be done by running | |
| # ``` | |
| # sudo visudo -f /etc/sudoers.d/github | |
| # ``` | |
| # on the guthub CI machine. | |
| sudo prlimit -lunlimited --pid $$ | |
| bash build_tools/ci/run_matmul_test.sh test_matmuls iree-install | |
| - name : E2E comparison of AIE to llvm-cpu | |
| run: | | |
| source .venv/bin/activate | |
| #install numpy (any version) | |
| pip install numpy | |
| bash build_tools/ci/cpu_comparison/run_test.sh test_aie_vs_cpu iree-install | |
| - name: Microkernel vs direct codegen -- performance and correctness | |
| run: | | |
| source .venv/bin/activate | |
| # See comment above about XRT memory | |
| sudo prlimit -lunlimited --pid $$ | |
| bash build_tools/ci/run_matmul_test.sh test_matmuls iree-install | |
| bash build_tools/ci/matmul_ukernel_vs_codegen_benchmark/run_test.sh test1 iree-install | |
| - name: Printing IR from aie2xclbin | |
| run: | | |
| source .venv/bin/activate | |
| bash build_tools/ci/print_ir_aie2xclbin/print_ir_aie2xclbin.sh iree-install/bin print_ir_aie2xclbin_results |