torch-nightly #416
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: torch-nightly | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # At 05:00 UTC (midnight EST) | |
| - cron: "0 5 * * *" | |
| push: | |
| paths: | |
| - "torch/**" | |
| - ".github/configurations/torch-base.yml" | |
| - ".github/configurations/torch-nccl.yml" | |
| - ".github/workflows/torch-nightly.yml" | |
| - ".github/workflows/torch.yml" | |
| - ".github/workflows/build.yml" | |
| jobs: | |
| get-nightly-info: | |
| name: | |
| Get Nightly Info | |
| runs-on: [ self-hosted, Linux ] | |
| outputs: | |
| pytorch-commit: ${{ steps.get-hash.outputs.pytorch-commit }} | |
| triton-commit: ${{ steps.get-hash.outputs.triton-commit }} | |
| torchvision-commit: ${{ steps.get-hash.outputs.torchvision-commit }} | |
| torchaudio-commit: ${{ steps.get-hash.outputs.torchaudio-commit }} | |
| version-string: ${{ steps.get-hash.outputs.version-string }} | |
| date: ${{ steps.get-date.outputs.date }} | |
| steps: | |
| - name: Get latest commit hashes | |
| id: get-hash | |
| run: | | |
| set -e; | |
| FORMAT_COMMIT_LINK() { | |
| printf '[`%.7s`](https://github.com/%s/tree/%s)\n' "$2" "$1" "$2"; | |
| }; | |
| LOG() { | |
| printf -- "$@" >> "$GITHUB_STEP_SUMMARY"; | |
| }; | |
| CLONE() { | |
| git clone --filter=blob:none --no-checkout --depth=1 \ | |
| "https://github.com/$1" \ | |
| "$2" > /dev/null 2> /dev/null && \ | |
| local COMMIT="$(git -C "$2" rev-parse HEAD)" && \ | |
| LOG 'Latest `%s` commit: %s\n' \ | |
| "$1" "$(FORMAT_COMMIT_LINK "$1" "$COMMIT")" && \ | |
| echo $COMMIT; | |
| }; | |
| GET_VERSION() { | |
| git -C "$1" show HEAD:version.txt 2> /dev/null; | |
| }; | |
| PYTORCH_COMMIT="$(CLONE pytorch/pytorch pytorch-git)"; | |
| PYTORCH_VERSION="$(GET_VERSION pytorch-git)"; | |
| TRITON_COMMIT_FILE=".ci/docker/ci_commit_pins/triton.txt"; | |
| TRITON_COMMIT="$(git -C pytorch-git show "HEAD:$TRITON_COMMIT_FILE" 2> /dev/null)"; | |
| rm -rf pytorch-git; | |
| LOG 'Corresponding `openai/triton` commit: %s\n' \ | |
| "$(FORMAT_COMMIT_LINK openai/triton "$TRITON_COMMIT")"; | |
| TORCHVISION_COMMIT="$(CLONE pytorch/vision torchvision-git)"; | |
| TORCHVISION_VERSION="$(GET_VERSION torchvision-git)"; | |
| rm -rf torchvision-git; | |
| TORCHAUDIO_COMMIT="$(CLONE pytorch/audio torchaudio-git)"; | |
| TORCHAUDIO_VERSION="$(GET_VERSION torchaudio-git)"; | |
| rm -rf torchaudio-git; | |
| echo "pytorch-commit=$PYTORCH_COMMIT" >> "$GITHUB_OUTPUT"; | |
| echo "triton-commit=$TRITON_COMMIT" >> "$GITHUB_OUTPUT"; | |
| echo "torchvision-commit=$TORCHVISION_COMMIT" >> "$GITHUB_OUTPUT"; | |
| echo "torchaudio-commit=$TORCHAUDIO_COMMIT" >> "$GITHUB_OUTPUT"; | |
| printf -- 'version-string=torch%s-vision%s-audio%s\n' \ | |
| "$PYTORCH_VERSION" "$TORCHVISION_VERSION" "$TORCHAUDIO_VERSION" \ | |
| >> "$GITHUB_OUTPUT"; | |
| - name: Get date | |
| id: get-date | |
| run: echo "date=$(date -u '+%y%m%d%H')" >> "$GITHUB_OUTPUT"; | |
| get-base-config: | |
| name: Get torch:base Config | |
| uses: ./.github/workflows/read-configuration.yml | |
| with: | |
| path: ./.github/configurations/torch-base.yml | |
| filter: del(.include) | |
| get-nccl-config: | |
| name: Get torch:nccl Config | |
| uses: ./.github/workflows/read-configuration.yml | |
| with: | |
| path: ./.github/configurations/torch-nccl.yml | |
| filter: del(.include) | |
| build-base: | |
| name: Build Nightly torch:base | |
| needs: | |
| - get-nightly-info | |
| - get-base-config | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.get-base-config.outputs.config) }} | |
| uses: ./.github/workflows/torch.yml | |
| secrets: inherit | |
| with: | |
| image-name: nightly-torch | |
| tag: ${{ format('base-{0}-cuda{1}-{2}-{3}', needs.get-nightly-info.outputs.date, matrix.cuda, matrix.os, needs.get-nightly-info.outputs.version-string) }} | |
| builder-base-image: nvidia/cuda:${{ matrix.cuda }}-devel-${{ matrix.os }} | |
| base-image: nvidia/cuda:${{ matrix.cuda }}-base-${{ matrix.os }} | |
| torch-version: ${{ needs.get-nightly-info.outputs.pytorch-commit }} | |
| torchvision-version: ${{ needs.get-nightly-info.outputs.torchvision-commit }} | |
| torchaudio-version: ${{ needs.get-nightly-info.outputs.torchaudio-commit }} | |
| triton-version: ${{ needs.get-nightly-info.outputs.triton-commit }} | |
| cache-key: base-cuda${{ matrix.cuda }}-${{ matrix.os }} | |
| build-extras: true | |
| build-nccl: | |
| name: Build Nightly torch:nccl | |
| needs: | |
| - get-nightly-info | |
| - get-nccl-config | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.get-nccl-config.outputs.config) }} | |
| uses: ./.github/workflows/torch.yml | |
| secrets: inherit | |
| with: | |
| image-name: nightly-torch | |
| tag: ${{ format('nccl-{0}-cuda{1}-{2}-nccl{3}-{4}', needs.get-nightly-info.outputs.date, matrix.image.cuda, matrix.image.os, matrix.image.nccl, needs.get-nightly-info.outputs.version-string ) }} | |
| builder-base-image: ghcr.io/coreweave/nccl-tests:${{ matrix.image.cuda }}-${{ matrix.image.cudnn }}-devel-${{ matrix.image.os }}-nccl${{ matrix.image.nccl }}-${{ matrix.image.nccl-tests-hash }} | |
| base-image: ghcr.io/coreweave/nccl-tests:${{ matrix.image.cuda }}-${{ matrix.image.cudnn }}-devel-${{ matrix.image.os }}-nccl${{ matrix.image.nccl }}-${{ matrix.image.nccl-tests-hash }} | |
| torch-version: ${{ needs.get-nightly-info.outputs.pytorch-commit }} | |
| torchvision-version: ${{ needs.get-nightly-info.outputs.torchvision-commit }} | |
| torchaudio-version: ${{ needs.get-nightly-info.outputs.torchaudio-commit }} | |
| triton-version: ${{ needs.get-nightly-info.outputs.triton-commit }} | |
| cache-key: nccl-cuda${{ matrix.image.cuda }}-${{ matrix.image.os }} | |
| build-extras: true |