slow_tests.yml

name: Non-regression tests

on:
  workflow_dispatch:
  schedule:
    - cron: '0 21 * * 0-5'  # every Sunday to Friday at 11pm CET (10pm winter time)
    - cron: '0 21 * * 6'  # every Saturday at 1am CET (midnight winter time)

concurrency:
  group: ${{ github.workflow }}

jobs:
  example-diff:
    name: Test examples differences
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash tests/ci/example_diff_tests.sh
  stable-diffusion:
    name: Test Stable Diffusion
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff  # run the job when the previous test job is done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash tests/ci/slow_tests_diffusers.sh
  deepspeed:
    name: Test DeepSpeed models
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff
      - stable-diffusion  # run the job when the previous test job is done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash tests/ci/slow_tests_deepspeed.sh
  multi-card:
    name: Test multi-card models
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff
      - deepspeed  # run the job when the previous test job is done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash tests/ci/slow_tests_8x.sh
  single-card:
    name: Test single-card models
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff
      - deepspeed
      - multi-card  # run the job when the previous test jobs are done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash tests/ci/slow_tests_1x.sh
  albert-xxl-single-card:
    name: Test single-card ALBERT XXL
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff
      - deepspeed
      - multi-card
      - single-card  # run the job when the previous test jobs are done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        if: github.event.schedule == '0 21 * * 6'
        uses: actions/checkout@v2
      - name: Pull image
        if: github.event.schedule == '0 21 * * 6'
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run test
        if: github.event.schedule == '0 21 * * 6'
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash tests/ci/albert_xxl_1x.sh
      - name: Warning
        if: github.event.schedule != '0 21 * * 6'
        run: echo "ALBERT XXL 1x is only tested on Saturdays."
  text-generation:
    name: Test text-generation example
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff
      - deepspeed
      - multi-card
      - single-card
      - albert-xxl-single-card  # run the job when the previous test jobs are done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            make slow_tests_text_generation_example TOKEN=${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }}
  trl:
    name: Test TRL integration
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff
      - deepspeed
      - multi-card
      - single-card
      - albert-xxl-single-card
      - text-generation  # run the job when the previous test jobs are done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash tests/ci/slow_tests_trl.sh
  sentence-transformers:
    name: Test Sentence Transformers integration
    if: ${{ !cancelled() && (success() || failure()) }}
    needs:
      - example-diff
      - deepspeed
      - multi-card
      - single-card
      - albert-xxl-single-card
      - text-generation
      - trl  # run the job when the previous test jobs are done
    runs-on:
      group: aws-dl1-24xlarge
    steps:
      - name: Checkout Optimum Habana
        uses: actions/checkout@v2
        with:
          repository: 'huggingface/optimum-habana'
          path: optimum-habana
      - name: Checkout Sentence Transformers
        uses: actions/checkout@v2
        with:
          repository: 'UKPLab/sentence-transformers'
          path: sentence-transformers
      - name: Pull image
        run: |
            docker pull vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest
      - name: Run tests
        run: |
            docker run \
            -v $PWD:/root/workspace \
            --workdir=/root/workspace \
            --runtime=habana \
            -e HABANA_VISIBLE_DEVICES=all \
            -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
            --cap-add=sys_nice \
            --net=host \
            --ipc=host \
            vault.habana.ai/gaudi-docker/1.17.0/ubuntu22.04/habanalabs/pytorch-installer-2.3.1:latest \
            /bin/bash optimum-habana/tests/ci/sentence_transformers.sh