Comparative Benchmarks #127
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2023 The IREE Authors | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| # | |
| # Comparative Benchmarks Workflow. | |
| name: Comparative Benchmarks | |
| on: | |
| schedule: | |
| # Scheduled to run at 09:00 UTC and 21:00 UTC. | |
| - cron: '0 09,21 * * *' | |
| workflow_dispatch: | |
| env: | |
| GCS_DIR: gs://iree-samples-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} | |
| jobs: | |
| build_gpu: | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=presubmit | |
| - environment=testing | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| CUDA_VERSION: 11.8 | |
| BENCHMARK_DEVICE: gpu | |
| BUILD_DIR: xla-build | |
| outputs: | |
| build-dir: ${{ env.BUILD_DIR }} | |
| build-dir-archive: ${{ steps.archive.outputs.build-dir-archive }} | |
| build-dir-gcs-artifact: ${{ steps.upload.outputs.build-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Building XLA" | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/cuda11.8-cudnn8.9.Dockerfile" \ | |
| --tag "cuda11.8-cudnn8.9" "oobi/build_tools/docker/context" | |
| mkdir "${BUILD_DIR}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "cuda11.8-cudnn8.9:latest" \ | |
| ./xla-hlo/benchmark/build_xla_tools.sh "${BENCHMARK_DEVICE}" "${BUILD_DIR}" "${CUDA_VERSION}" | |
| - name: "Creating build dir archive" | |
| id: archive | |
| env: | |
| BUILD_DIR_ARCHIVE: ${{ env.BUILD_DIR }}.tar.zst | |
| run: | | |
| tar -I 'zstd -T0' \ | |
| -cf ${BUILD_DIR_ARCHIVE} ${BUILD_DIR} | |
| echo "build-dir-archive=${BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Uploading build dir archive" | |
| id: upload | |
| env: | |
| BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.build-dir-archive }} | |
| BUILD_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.build-dir-archive }} | |
| run: | | |
| gcloud storage cp "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR_GCS_ARTIFACT}" | |
| echo "build-dir-gcs-artifact=${BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| benchmark_gpu: | |
| needs: [build_gpu] | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=presubmit | |
| - environment=testing | |
| - gpu | |
| - os-family=Linux | |
| env: | |
| TF_VERSION: 2.12.0 | |
| LOCAL_OUTPUT_DIR: results-dir | |
| TF_RESULTS_JSON: tf-xla.json | |
| JAX_RESULTS_JSON: jax-xla.json | |
| HLO_RESULTS_JSON: hlo-xla.json | |
| PT_RESULTS_JSON: pt-inductor.json | |
| BENCHMARK_DEVICE: gpu | |
| GCS_UPLOAD_ROOT_DIR: "gs://comparative-benchmark-artifacts" | |
| BUILD_DIR: ${{ needs.build_gpu.outputs.build-dir }} | |
| BUILD_DIR_ARCHIVE: ${{ needs.build_gpu.outputs.build-dir-archive }} | |
| BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_gpu.outputs.build-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Setup" | |
| run: | | |
| echo "GCS_UPLOAD_DIR=${GCS_UPLOAD_ROOT_DIR}/${BENCHMARK_DEVICE}_$(date +'%Y-%m-%d').$(date +'%s')" >> $GITHUB_ENV | |
| mkdir "${LOCAL_OUTPUT_DIR}" | |
| - name: "Downloading build dir archive" | |
| run: gcloud storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}" | |
| - name: "Extracting build dir archive" | |
| run: tar -xf "${BUILD_DIR_ARCHIVE}" | |
| - name: "Benchmarking HLO/XLA:GPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/cuda11.8-cudnn8.9.Dockerfile" \ | |
| --tag "cuda11.8-cudnn8.9" "oobi/build_tools/docker/context" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${HLO_RESULTS_JSON}" | |
| docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "cuda11.8-cudnn8.9:latest" \ | |
| ./xla-hlo/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${BUILD_DIR}/hlo_runner_main" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" | |
| - name: "Benchmarking PyTorch/Inductor:GPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/cuda11.8-cudnn8.9.Dockerfile" \ | |
| --tag "cuda11.8-cudnn8.9" "oobi/build_tools/docker/context" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${PT_RESULTS_JSON}" | |
| docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "cuda11.8-cudnn8.9:latest" \ | |
| ./iree-torch/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" | |
| - name: "Benchmarking TF/XLA:GPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/cuda11.8-cudnn8.9.Dockerfile" \ | |
| --tag "cuda11.8-cudnn8.9" "oobi/build_tools/docker/context" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${TF_RESULTS_JSON}" | |
| docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "cuda11.8-cudnn8.9:latest" \ | |
| ./iree-tf/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${TF_VERSION}" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" | |
| - name: "Benchmarking JAX/XLA:GPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/cuda11.8-cudnn8.9.Dockerfile" \ | |
| --tag "cuda11.8-cudnn8.9" "oobi/build_tools/docker/context" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${JAX_RESULTS_JSON}" | |
| docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "cuda11.8-cudnn8.9:latest" \ | |
| ./iree-jax/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" | |
| benchmark_cpu: | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=presubmit | |
| - environment=testing | |
| - machine-type=c2-standard-16 | |
| - os-family=Linux | |
| env: | |
| BUILD_DIR: xla-build | |
| TF_VERSION: 2.12.0 | |
| LOCAL_OUTPUT_DIR: results-dir | |
| TF_RESULTS_JSON: tf-xla.json | |
| JAX_RESULTS_JSON: jax-xla.json | |
| HLO_RESULTS_JSON: hlo-xla.json | |
| PT_RESULTS_JSON: pt-inductor.json | |
| BENCHMARK_DEVICE: cpu | |
| GCS_UPLOAD_ROOT_DIR: "gs://comparative-benchmark-artifacts" | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Setup" | |
| run: | | |
| echo "GCS_UPLOAD_DIR=${GCS_UPLOAD_ROOT_DIR}/${BENCHMARK_DEVICE}_$(date +'%Y-%m-%d').$(date +'%s')" >> $GITHUB_ENV | |
| mkdir "${LOCAL_OUTPUT_DIR}" | |
| - name: "Benchmarking HLO/XLA:CPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/base.Dockerfile" \ | |
| --tag "base" "oobi/build_tools/docker/context" | |
| mkdir "${BUILD_DIR}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "base:latest" \ | |
| ./xla-hlo/benchmark/build_xla_tools.sh "${BENCHMARK_DEVICE}" "${BUILD_DIR}" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${HLO_RESULTS_JSON}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "base:latest" \ | |
| ./xla-hlo/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${BUILD_DIR}/run_hlo_module" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" | |
| - name: "Benchmarking PyTorch/Inductor:CPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/base.Dockerfile" \ | |
| --tag "base" "oobi/build_tools/docker/context" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${PT_RESULTS_JSON}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "base:latest" \ | |
| ./iree-torch/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" | |
| - name: "Benchmarking TF/XLA:CPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/base.Dockerfile" \ | |
| --tag "base" "oobi/build_tools/docker/context" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${TF_RESULTS_JSON}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "base:latest" \ | |
| ./iree-tf/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${TF_VERSION}" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" | |
| - name: "Benchmarking JAX/XLA:CPU" | |
| timeout-minutes: 360 | |
| run: | | |
| docker build --file "oobi/build_tools/docker/dockerfiles/base.Dockerfile" \ | |
| --tag "base" "oobi/build_tools/docker/context" | |
| RESULTS_PATH="${LOCAL_OUTPUT_DIR}/${JAX_RESULTS_JSON}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "base:latest" \ | |
| ./iree-jax/benchmark/benchmark_all.sh "${BENCHMARK_DEVICE}" "${RESULTS_PATH}" | |
| cat "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${GCS_UPLOAD_DIR}/" |