From 8ff38100dda59464ad794f1cd1ec2aa982c4e491 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 19 Dec 2023 20:50:51 -0800 Subject: [PATCH 1/8] Enable Python 3.12 build for torchdata --- .github/workflows/_build_test_upload.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index bec5b41c9..488551956 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -70,6 +70,7 @@ jobs: - 3.9 - "3.10" - "3.11" + - "3.12" - pure exclude: - os: macos-latest @@ -318,6 +319,7 @@ jobs: - 3.9 - "3.10" - "3.11" + - "3.12" steps: - name: Checkout Source Repository uses: actions/checkout@v3 @@ -366,12 +368,7 @@ jobs: shell: bash -l {0} run: | source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - - if [[ ${{ matrix.python-version }} = "3.11" ]]; then - conda create -y -p "${CONDA_ENV_PATH}" python=3.8 - else - conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} - fi + conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} - name: Build TorchData for Conda shell: bash -l {0} env: @@ -384,12 +381,7 @@ jobs: source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" fi conda activate "${CONDA_ENV_PATH}" - if [[ "${PYTHON_VERSION}" = "3.11" ]]; then - conda install -yq conda-build -c malfet -c conda-forge - export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet" - else - conda install -yq conda-build - fi + conda install -yq conda-build packaging/build_conda.sh conda index ./conda-bld - name: Upload Conda Package to Github From 1c9756a97c530e9f9610f151e14753ea4b081573 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 19 Dec 2023 21:17:59 -0800 Subject: [PATCH 2/8] Minor push to run all builds --- .github/workflows/_build_test_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index 488551956..980458f21 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -192,7 +192,7 @@ jobs: fi pip3 install pkginfo for pkg in dist/torchdata*.whl; do - echo "PkgInfo of $pkg:" + echo "PkgInfo of ${pkg}:" pkginfo $pkg if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then auditwheel show $pkg From e32e58a0c049f03a1975a9d9fb5c2936e4bd51bd Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 19 Dec 2023 22:16:38 -0800 Subject: [PATCH 3/8] Build conda 3.12 with conda-build in 3.9 --- .github/workflows/_build_test_upload.yml | 16 +++++++++++++--- packaging/build_wheel.sh | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index 980458f21..f3b0bbe43 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -192,7 +192,7 @@ jobs: fi pip3 install pkginfo for pkg in dist/torchdata*.whl; do - echo "PkgInfo of ${pkg}:" + echo "PkgInfo of $pkg:" pkginfo $pkg if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then auditwheel show $pkg @@ -368,7 +368,12 @@ jobs: shell: bash -l {0} run: | source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} + + if [[ ${{ matrix.python-version }} = "3.12" ]]; then + conda create -y -p "${CONDA_ENV_PATH}" python=3.9 + else + conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} + fi - name: Build TorchData for Conda shell: bash -l {0} env: @@ -381,7 +386,12 @@ jobs: source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" fi conda activate "${CONDA_ENV_PATH}" - conda install -yq conda-build + if [[ "${PYTHON_VERSION}" = "3.12" ]]; then + conda install -yq conda-build -c malfet -c conda-forge + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet" + else + conda install -yq conda-build + fi packaging/build_conda.sh conda index ./conda-bld - name: Upload Conda Package to Github diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 7a965dd02..8e4aab4e1 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -16,7 +16,7 @@ export BUILD_TYPE="wheel" export SOURCE_ROOT_DIR="$PWD" setup_env -pip_install future wheel +pip_install future wheel distutils setup_pip_pytorch_version git submodule update --init --recursive From 51a295c3c71c37f3e5ec1473e07e45d01d18a343 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 19 Dec 2023 22:23:43 -0800 Subject: [PATCH 4/8] Install setuptools for 3.12 --- packaging/build_wheel.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 8e4aab4e1..6d7f7a24f 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -16,7 +16,8 @@ export BUILD_TYPE="wheel" export SOURCE_ROOT_DIR="$PWD" setup_env -pip_install future wheel distutils +# Python 3.12 does comes with distutils anymore +pip_install future wheel setuptools setup_pip_pytorch_version git submodule update --init --recursive From cd8857ac00c8944e1ff1ab2018b288bf34ffb111 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 19 Dec 2023 23:12:13 -0800 Subject: [PATCH 5/8] Use the correct conda env --- .github/workflows/_build_test_upload.yml | 37 +++++++++++------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index f3b0bbe43..d3b4079f6 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -108,12 +108,13 @@ jobs: bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}" rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> "$GITHUB_PATH" + echo "CONDA_EXE=${MINICONDA_INSTALL_PATH_MACOS}/bin/conda" >> "$GITHUB_ENV" - name: Setup Python ${{ matrix.python-version }} on MacOS if: ${{ startsWith( matrix.os, 'macos' ) }} shell: bash -l {0} run: | - conda init bash - conda create -y --name wheel_build_env python=${{ matrix.python-version }} + $CONDA_EXE init bash + $CONDA_EXE create -y --name wheel_build_env python=${{ matrix.python-version }} - name: Setup msbuild on Windows if: startsWith( matrix.os, 'windows' ) uses: microsoft/setup-msbuild@v1.1 @@ -138,7 +139,7 @@ jobs: fi else if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env + $CONDA_EXE activate wheel_build_env fi pip install cmake ninja echo "/home/runner/.local/bin" >> $GITHUB_PATH @@ -170,8 +171,8 @@ jobs: fi fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - conda run -n wheel_build_env packaging/build_wheel.sh + $CONDA_EXE activate wheel_build_env + $CONDA_EXE run -n wheel_build_env packaging/build_wheel.sh else packaging/build_wheel.sh fi @@ -188,7 +189,7 @@ jobs: pip3 install auditwheel fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env + $CONDA_EXE activate wheel_build_env fi pip3 install pkginfo for pkg in dist/torchdata*.whl; do @@ -215,8 +216,8 @@ jobs: source packaging/manylinux/python_helper.sh fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - conda run -n wheel_build_env pip3 install dist/torchdata*.whl + $CONDA_EXE activate wheel_build_env + $CONDA_EXE run -n wheel_build_env pip3 install dist/torchdata*.whl else pip3 install dist/torchdata*.whl fi @@ -232,8 +233,8 @@ jobs: source packaging/manylinux/python_helper.sh fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - conda run -n wheel_build_env python test/smoke_test/smoke_test.py + $CONDA_EXE activate wheel_build_env + $CONDA_EXE run -n wheel_build_env python test/smoke_test/smoke_test.py else if ${{ matrix.python-version == 'pure' }}; then python test/smoke_test/smoke_test.py --no-s3 @@ -362,6 +363,7 @@ jobs: bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}" rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh" echo "${MINICONDA_INSTALL_PATH}/bin" >> "$GITHUB_PATH" + echo "CONDA_EXE=${MINICONDA_INSTALL_PATH}/bin/conda" >> "$GITHUB_ENV" ${MINICONDA_INSTALL_PATH}/bin/conda init bash - name: Create Conda Env on MacOS and Linux if: ${{ ! startsWith( matrix.os, 'windows' ) }} @@ -370,9 +372,9 @@ jobs: source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" if [[ ${{ matrix.python-version }} = "3.12" ]]; then - conda create -y -p "${CONDA_ENV_PATH}" python=3.9 + $CONDA_EXE create -y -p "${CONDA_ENV_PATH}" python=3.9 else - conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} + $CONDA_EXE create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} fi - name: Build TorchData for Conda shell: bash -l {0} @@ -385,15 +387,10 @@ jobs: if ${{ ! startsWith( matrix.os, 'windows' ) }}; then source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" fi - conda activate "${CONDA_ENV_PATH}" - if [[ "${PYTHON_VERSION}" = "3.12" ]]; then - conda install -yq conda-build -c malfet -c conda-forge - export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet" - else - conda install -yq conda-build - fi + $CONDA_EXE activate "${CONDA_ENV_PATH}" + $CONDA_EXE install -yq conda-build packaging/build_conda.sh - conda index ./conda-bld + $CONDA_EXE index ./conda-bld - name: Upload Conda Package to Github if: always() uses: actions/upload-artifact@v3 From fb406e1191e5e492a40fd8f6c3f9dff19e6e02f1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 19 Dec 2023 23:39:10 -0800 Subject: [PATCH 6/8] Figure it out --- .github/workflows/_build_test_upload.yml | 41 ++++++++++++------------ .github/workflows/build-conda-m1.yml | 2 +- .github/workflows/build-wheels-m1.yml | 2 +- .github/workflows/build_conda_linux.yml | 4 +-- .github/workflows/build_wheels_linux.yml | 3 +- packaging/build_wheel.sh | 3 +- 6 files changed, 26 insertions(+), 29 deletions(-) diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index d3b4079f6..bec5b41c9 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -70,7 +70,6 @@ jobs: - 3.9 - "3.10" - "3.11" - - "3.12" - pure exclude: - os: macos-latest @@ -108,13 +107,12 @@ jobs: bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}" rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> "$GITHUB_PATH" - echo "CONDA_EXE=${MINICONDA_INSTALL_PATH_MACOS}/bin/conda" >> "$GITHUB_ENV" - name: Setup Python ${{ matrix.python-version }} on MacOS if: ${{ startsWith( matrix.os, 'macos' ) }} shell: bash -l {0} run: | - $CONDA_EXE init bash - $CONDA_EXE create -y --name wheel_build_env python=${{ matrix.python-version }} + conda init bash + conda create -y --name wheel_build_env python=${{ matrix.python-version }} - name: Setup msbuild on Windows if: startsWith( matrix.os, 'windows' ) uses: microsoft/setup-msbuild@v1.1 @@ -139,7 +137,7 @@ jobs: fi else if ${{ startsWith( matrix.os, 'macos' ) }}; then - $CONDA_EXE activate wheel_build_env + conda activate wheel_build_env fi pip install cmake ninja echo "/home/runner/.local/bin" >> $GITHUB_PATH @@ -171,8 +169,8 @@ jobs: fi fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - $CONDA_EXE activate wheel_build_env - $CONDA_EXE run -n wheel_build_env packaging/build_wheel.sh + conda activate wheel_build_env + conda run -n wheel_build_env packaging/build_wheel.sh else packaging/build_wheel.sh fi @@ -189,7 +187,7 @@ jobs: pip3 install auditwheel fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - $CONDA_EXE activate wheel_build_env + conda activate wheel_build_env fi pip3 install pkginfo for pkg in dist/torchdata*.whl; do @@ -216,8 +214,8 @@ jobs: source packaging/manylinux/python_helper.sh fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - $CONDA_EXE activate wheel_build_env - $CONDA_EXE run -n wheel_build_env pip3 install dist/torchdata*.whl + conda activate wheel_build_env + conda run -n wheel_build_env pip3 install dist/torchdata*.whl else pip3 install dist/torchdata*.whl fi @@ -233,8 +231,8 @@ jobs: source packaging/manylinux/python_helper.sh fi if ${{ startsWith( matrix.os, 'macos' ) }}; then - $CONDA_EXE activate wheel_build_env - $CONDA_EXE run -n wheel_build_env python test/smoke_test/smoke_test.py + conda activate wheel_build_env + conda run -n wheel_build_env python test/smoke_test/smoke_test.py else if ${{ matrix.python-version == 'pure' }}; then python test/smoke_test/smoke_test.py --no-s3 @@ -320,7 +318,6 @@ jobs: - 3.9 - "3.10" - "3.11" - - "3.12" steps: - name: Checkout Source Repository uses: actions/checkout@v3 @@ -363,7 +360,6 @@ jobs: bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}" rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh" echo "${MINICONDA_INSTALL_PATH}/bin" >> "$GITHUB_PATH" - echo "CONDA_EXE=${MINICONDA_INSTALL_PATH}/bin/conda" >> "$GITHUB_ENV" ${MINICONDA_INSTALL_PATH}/bin/conda init bash - name: Create Conda Env on MacOS and Linux if: ${{ ! startsWith( matrix.os, 'windows' ) }} @@ -371,10 +367,10 @@ jobs: run: | source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - if [[ ${{ matrix.python-version }} = "3.12" ]]; then - $CONDA_EXE create -y -p "${CONDA_ENV_PATH}" python=3.9 + if [[ ${{ matrix.python-version }} = "3.11" ]]; then + conda create -y -p "${CONDA_ENV_PATH}" python=3.8 else - $CONDA_EXE create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} + conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} fi - name: Build TorchData for Conda shell: bash -l {0} @@ -387,10 +383,15 @@ jobs: if ${{ ! startsWith( matrix.os, 'windows' ) }}; then source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" fi - $CONDA_EXE activate "${CONDA_ENV_PATH}" - $CONDA_EXE install -yq conda-build + conda activate "${CONDA_ENV_PATH}" + if [[ "${PYTHON_VERSION}" = "3.11" ]]; then + conda install -yq conda-build -c malfet -c conda-forge + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet" + else + conda install -yq conda-build + fi packaging/build_conda.sh - $CONDA_EXE index ./conda-bld + conda index ./conda-bld - name: Upload Conda Package to Github if: always() uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml index b61649b87..c565b7b08 100644 --- a/.github/workflows/build-conda-m1.yml +++ b/.github/workflows/build-conda-m1.yml @@ -38,6 +38,6 @@ jobs: env-var-script: packaging/env-var-script.txt smoke-test-script: test/smoke_test/smoke_test.py runner-type: macos-m1-12 - trigger-event: dev + trigger-event: ${{ github.event_name }} secrets: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml index d66875933..2fdb5f6e5 100644 --- a/.github/workflows/build-wheels-m1.yml +++ b/.github/workflows/build-wheels-m1.yml @@ -37,7 +37,7 @@ jobs: env-var-script: packaging/env-var-script.txt runner-type: macos-m1-12 smoke-test-script: test/smoke_test/smoke_test.py - trigger-event: dev + trigger-event: ${{ github.event_name }} secrets: AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml index 76d3b526d..d2afb025f 100644 --- a/.github/workflows/build_conda_linux.yml +++ b/.github/workflows/build_conda_linux.yml @@ -43,8 +43,6 @@ jobs: smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} env-var-script: packaging/env-var-script.txt - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development + trigger-event: ${{ github.event_name }} secrets: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 1f1c9d9ef..b179929b4 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -41,8 +41,7 @@ jobs: smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} env-var-script: packaging/env-var-script.txt - # Using "development" as trigger event so these binaries are not uploaded to official channels yet - trigger-event: development + trigger-event: ${{ github.event_name }} secrets: AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 6d7f7a24f..7a965dd02 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -16,8 +16,7 @@ export BUILD_TYPE="wheel" export SOURCE_ROOT_DIR="$PWD" setup_env -# Python 3.12 does comes with distutils anymore -pip_install future wheel setuptools +pip_install future wheel setup_pip_pytorch_version git submodule update --init --recursive From 42b00f54e838ba4fa673845707e8a0d89b56ecf1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 19 Dec 2023 23:42:25 -0800 Subject: [PATCH 7/8] Clean up obsolete workflows --- .github/workflows/_build_test_upload.yml | 555 ----------------------- .github/workflows/nightly_release.yml | 20 - .github/workflows/pull_release.yml | 22 - .github/workflows/release.yml | 24 - .github/workflows/test_release.yml | 36 -- 5 files changed, 657 deletions(-) delete mode 100644 .github/workflows/_build_test_upload.yml delete mode 100644 .github/workflows/nightly_release.yml delete mode 100644 .github/workflows/pull_release.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/test_release.yml diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml deleted file mode 100644 index bec5b41c9..000000000 --- a/.github/workflows/_build_test_upload.yml +++ /dev/null @@ -1,555 +0,0 @@ -name: Build, Test and Upload Wheel - -on: - workflow_call: - inputs: - branch: - required: true - type: string - pre_dev_release: - required: true - type: boolean - pytorch_version: - required: true - type: string - do-upload: - required: false - default: true - type: boolean - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: - required: true - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: - required: true - PYPI_TOKEN: - required: false - CONDA_PYTORCHBOT_TOKEN: - required: false - CONDA_TEST_PYTORCHBOT_TOKEN: - required: false - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: - required: false - -jobs: - get_release_type: - runs-on: ubuntu-latest - outputs: - type: ${{ steps.get_release_type.outputs.type }} - steps: - - name: Get Release Type - run: | - if [[ "${{ inputs.branch }}" == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then - RELEASE_TYPE=official - elif [[ "${{ inputs.branch }}" == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then - RELEASE_TYPE=test - else - if [[ "${{ github.base_ref }}" == release/* ]]; then - RELEASE_TYPE=test - else - RELEASE_TYPE=nightly - fi - fi - echo "Release Type: $RELEASE_TYPE" - echo "type=$RELEASE_TYPE" >> $GITHUB_OUTPUT - id: get_release_type - - wheel_build_test: - needs: get_release_type - runs-on: ${{ matrix.os }} - container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - - macos-m1-12 - python-version: - - 3.8 - - 3.9 - - "3.10" - - "3.11" - - pure - exclude: - - os: macos-latest - python-version: pure - - os: windows-latest - python-version: pure - - os: macos-m1-12 - python-version: pure - steps: - - name: Checkout Source Repository - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Setup Python ${{ matrix.python-version }} for Windows - if: ${{ startsWith( matrix.os, 'windows' ) }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Add temp runner environment variables - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}" - - name: Install Conda on MacOS - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - mkdir -p "${MINICONDA_INSTALL_PATH_MACOS}" - if ${{ startsWith( matrix.os, 'macos-m1' ) }}; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - else - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - fi - bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}" - rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> "$GITHUB_PATH" - - name: Setup Python ${{ matrix.python-version }} on MacOS - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - conda init bash - conda create -y --name wheel_build_env python=${{ matrix.python-version }} - - name: Setup msbuild on Windows - if: startsWith( matrix.os, 'windows' ) - uses: microsoft/setup-msbuild@v1.1 - - name: Set up Visual Studio shell - if: startsWith( matrix.os, 'windows' ) - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - name: Install Build Dependency - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version != 'pure' }}; then - source packaging/manylinux/python_helper.sh - yum -y install ninja-build zlib-static - # Docker path is /__w by default - export WORKSPACE="/__w" - # Install static OpenSSL/libcrypto library - ./packaging/manylinux/install_openssl_curl.sh - fi - else - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - fi - pip install cmake ninja - echo "/home/runner/.local/bin" >> $GITHUB_PATH - fi - - name: Install PyTorch and Build TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} - BUILD_S3: ${{ matrix.python-version != 'pure' }} - run: | - set -ex - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - # Use Python 3.8 for pure python build - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - # Docker path is /__w by default - export WORKSPACE="/__w" - # See: https://github.com/actions/checkout/issues/760 - git config --global --add safe.directory "$WORKSPACE/data/data" - if ${{ matrix.python-version != 'pure' }}; then - # AWSSDK uses $CMAKE_PREFIX_PATH to find openssl - export OPENSSL_ROOT_DIR="$WORKSPACE/ssl" - export CURL_ROOT_DIR="$WORKSPACE/curl" - export CMAKE_PREFIX_PATH="$OPENSSL_ROOT_DIR:$CURL_ROOT_DIR:$CMAKE_PREFIX_PATH" - export STATIC_DEPS=TRUE - fi - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - conda run -n wheel_build_env packaging/build_wheel.sh - else - packaging/build_wheel.sh - fi - - name: Validate TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - pip3 install auditwheel - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - fi - pip3 install pkginfo - for pkg in dist/torchdata*.whl; do - echo "PkgInfo of $pkg:" - pkginfo $pkg - if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then - auditwheel show $pkg - auditwheel repair $pkg --plat manylinux2014_x86_64 -w wheelhouse - fi - done - if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then - mv dist temp - mv wheelhouse dist - fi - - name: Install TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - conda run -n wheel_build_env pip3 install dist/torchdata*.whl - else - pip3 install dist/torchdata*.whl - fi - - name: Run Smoke Tests - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - conda run -n wheel_build_env python test/smoke_test/smoke_test.py - else - if ${{ matrix.python-version == 'pure' }}; then - python test/smoke_test/smoke_test.py --no-s3 - else - python test/smoke_test/smoke_test.py - fi - fi - - name: Upload Wheels to Github - if: always() - uses: actions/upload-artifact@v3 - with: - name: torchdata-artifact - path: dist/torchdata*.whl - - wheel_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, wheel_build_test] - runs-on: ubuntu-latest - environment: pytorchbot-env - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: torchdata-artifact - - name: Determine if Wheel Uploading is needed - run: | - upload=false - for txt in torchdata*.whl; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Wheels - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh torchdata*.whl - - name: Upload Wheels to S3 Storage - if: steps.trigger_upload.outputs.value == 'true' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - run: | - if [[ ${{ inputs.branch }} == 'main' ]]; then - S3_PATH=s3://pytorch/whl/nightly/ - else - S3_PATH=s3://pytorch/whl/test/ - fi - pip3 install awscli - set -x - for pkg in torchdata*.whl; do - aws s3 cp "$pkg" "$S3_PATH" --acl public-read - done - - name: Upload Official Wheels to PYPI - if: | - steps.trigger_upload.outputs.value == 'true' && - needs.get_release_type.outputs.type == 'official' - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: | - pip3 install twine - python -m twine upload \ - --username __token__ \ - --password "$PYPI_TOKEN" \ - torchdata*.whl - - conda_build_test: - needs: get_release_type - runs-on: ${{ matrix.os }} - container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - - macos-m1-12 - python-version: - - 3.8 - - 3.9 - - "3.10" - - "3.11" - steps: - - name: Checkout Source Repository - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Determine if build AWSSDK - shell: bash -l {0} - run: | - if ${{ startsWith( matrix.os, 'windows' ) }}; then - BUILD_S3=0 - else - BUILD_S3=1 - fi - echo "value=$BUILD_S3" >> $GITHUB_OUTPUT - id: build_s3 - - name: Add temp runner environment variables - shell: bash -l {0} - run: | - echo "MINICONDA_INSTALL_PATH=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}" - echo "CONDA_ENV_PATH=${RUNNER_TEMP}/conda_build_env" >> "${GITHUB_ENV}" - - name: Create Conda Env on Windows - if: ${{ startsWith( matrix.os, 'windows' ) }} - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: ${{ matrix.python-version }} - activate-environment: ${{ env.CONDA_ENV_PATH }} - - name: Install Conda on MacOS and Linux - if: ${{ ! startsWith( matrix.os, 'windows' ) }} - shell: bash -l {0} - run: | - mkdir -p "${MINICONDA_INSTALL_PATH}" - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o "${MINICONDA_INSTALL_PATH}/miniconda.sh" - elif ${{ startsWith( matrix.os, 'macos-m1' ) }}; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH}/miniconda.sh" - else - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH}/miniconda.sh" - fi - bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}" - rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh" - echo "${MINICONDA_INSTALL_PATH}/bin" >> "$GITHUB_PATH" - ${MINICONDA_INSTALL_PATH}/bin/conda init bash - - name: Create Conda Env on MacOS and Linux - if: ${{ ! startsWith( matrix.os, 'windows' ) }} - shell: bash -l {0} - run: | - source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - - if [[ ${{ matrix.python-version }} = "3.11" ]]; then - conda create -y -p "${CONDA_ENV_PATH}" python=3.8 - else - conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} - fi - - name: Build TorchData for Conda - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} - BUILD_S3: ${{ steps.build_s3.outputs.value }} - run: | - set -ex - if ${{ ! startsWith( matrix.os, 'windows' ) }}; then - source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - fi - conda activate "${CONDA_ENV_PATH}" - if [[ "${PYTHON_VERSION}" = "3.11" ]]; then - conda install -yq conda-build -c malfet -c conda-forge - export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet" - else - conda install -yq conda-build - fi - packaging/build_conda.sh - conda index ./conda-bld - - name: Upload Conda Package to Github - if: always() - uses: actions/upload-artifact@v3 - with: - name: torchdata-artifact - path: conda-bld/*/torchdata-*.tar.bz2 - - conda_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, conda_build_test] - runs-on: ubuntu-latest - container: continuumio/miniconda3 - environment: pytorchbot-env - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: torchdata-artifact - - name: Determine if Conda Uploading is needed - run: | - upload=false - for pkg in ./*/torchdata-*.tar.bz2; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Conda Package - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh ./*/torchdata-*.tar.bz2 - - name: Upload Packages to Conda - if: steps.trigger_upload.outputs.value == 'true' - shell: bash -l {0} - env: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} - CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} - run: | - conda create -y --name conda_upload_env - conda activate conda_upload_env - - conda install -yq anaconda-client - conda install -c conda-forge -yq jq - - if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then - CONDA_CHANNEL=pytorch - CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN} - elif [[ ${{ needs.get_release_type.outputs.type }} == 'test' ]]; then - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_TEST_PYTORCHBOT_TOKEN} - else - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_NIGHTLY_PYTORCHBOT_TOKEN} - fi - - if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then - # Loop over all platforms [win-64, osx-64, linux-64] - for subdir in $(find . -type f -name '*torchdata*.tar.bz2' | sed -r 's|/[^/]+$||' | uniq | cut -f2 -d/) ; do - version="" - # Find existing conda packages on pytorch nightly - for val in $(conda search --json torchdata --channel=$CONDA_CHANNEL --subdir=$subdir | jq -r '.[][] | .version, .build'); do - if [[ -z $version ]]; then - version=$val - else - build=$val - # Check if a new conda package built based on the same Python version and platform exists - for new_package in ./$subdir/torchdata-*-$build.tar.bz2; do - if [[ -f "$new_package" ]]; then - echo "Removing $CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - anaconda -t "${CONDA_TOKEN}" remove -f "$CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - break - fi - done - version="" - build="" - fi - done - # Upload new conda packages per subdir - anaconda -t "${CONDA_TOKEN}" upload ./$subdir/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - done - else - anaconda -t "${CONDA_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - fi - - build_docs: - if: | - always() && inputs.branch != '' && - ( needs.wheel_upload.outputs.upload == 'true' || needs.conda_upload.outputs.upload == 'true' ) - needs: [get_release_type, wheel_upload, conda_upload] - runs-on: ubuntu-latest - steps: - - name: Setup Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Install Dependencies - run: | - echo `python3 --version` - sudo apt-get install -y python-setuptools - python3 -m pip install --upgrade pip - python3 -m pip install setuptools - python3 -m pip install matplotlib - sudo apt-get install -y yarn - - name: Install PyTorch & TorchData - run: | - pip3 install numpy - # Add version requirement to PyTorch except nightly release - if [[ -z "${{ inputs.pytorch_version }}" ]]; then - PYTORCH_VERSION=torch - else - PYTORCH_VERSION=torch==${{ inputs.pytorch_version }} - fi - - PIP_CHANNEL=${{ needs.get_release_type.outputs.type }} - if [[ $PIP_CHANNEL == 'official' ]]; then - pip3 install "$PYTORCH_VERSION" -f https://download.pytorch.org/whl/torch_stable.html - else - pip3 install --pre "$PYTORCH_VERSION" -f "https://download.pytorch.org/whl/$PIP_CHANNEL/torch_$PIP_CHANNEL.html" - fi - - pip3 install -r requirements.txt - pip3 install . - - name: Check env - run: echo `which spinx-build` - - name: Build the docset - run: | - cd ./docs - sudo apt-get install -y graphviz - pip3 install -r requirements.txt - make html - cd .. - - name: Export Target Folder - run: | - TARGET_FOLDER=${{ inputs.branch }} - if [[ $TARGET_FOLDER == release/* ]]; then - TARGET_FOLDER=${TARGET_FOLDER:8} - elif [[ $TARGET_FOLDER == tags/* ]]; then - TARGET_FOLDER=${TARGET_FOLDER:5} - elif [[ $TARGET_FOLDER == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then - if [[ $TARGET_FOLDER == v*.*.* ]]; then - TARGET_FOLDER=${TARGET_FOLDER%.*} - fi - TARGET_FOLDER=${TARGET_FOLDER:1} - fi - echo "value=$TARGET_FOLDER" >> $GITHUB_OUTPUT - id: target_folder - - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs/build/html # The folder the action should deploy. - target-folder: ${{ steps.target_folder.outputs.value }} # The destination folder the action should deploy to. diff --git a/.github/workflows/nightly_release.yml b/.github/workflows/nightly_release.yml deleted file mode 100644 index 02771077a..000000000 --- a/.github/workflows/nightly_release.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Push Nightly Release - -on: - workflow_dispatch: - schedule: - - cron: 00 11 * * * - -jobs: - build_test_upload: - if: | - github.repository == 'pytorch/data' && github.ref_name == 'main' - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "main" - pre_dev_release: true - pytorch_version: "" - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/pull_release.yml b/.github/workflows/pull_release.yml deleted file mode 100644 index c73b12f04..000000000 --- a/.github/workflows/pull_release.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Test Release Pipelines - -on: - workflow_dispatch: - pull_request: - paths: - - .github/workflows/pull_release.yml - - .github/workflows/_build_test_upload.yml - -jobs: - build_test_upload: - if: github.repository == 'pytorch/data' - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "" - pre_dev_release: true - pytorch_version: "" - do-upload: false - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: "" - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: "" - CONDA_TEST_PYTORCHBOT_TOKEN: "" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 10494136f..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Push Official Release - -on: - # [ Note: Manually Trigger the Workflow ] - # 1. Go to Actions under pytorch/data repo - # 2. In the left sidebar, click the workflow you want to run - # 3. Above the list of workflow runs, select Run workflow - # 4. Use the Branch dropdown to select the release/* branch - # 5. Click Run workflow - workflow_dispatch: - -jobs: - build_test_upload: - if: github.repository == 'pytorch/data' && github.ref_type == 'tag' - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "" - pre_dev_release: false - pytorch_version: "" - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml deleted file mode 100644 index 811118139..000000000 --- a/.github/workflows/test_release.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Push Test Release - -on: - # [ Note: Manually Trigger the Workflow ] - # 1. Go to Actions under pytorch/data repo - # 2. In the left sidebar, click the workflow you want to run - # 3. Above the list of workflow runs, select Run workflow - # 4. Use the Branch dropdown to select the release/* branch - # 5. Click Run workflow - workflow_dispatch: - # Automatically trigger test/official release - # Requred Feature of GHA: Run schedule on specific branch - # Otherwise, all changes for release need to be landed into main branch - # See: https://github.community/t/scheduled-builds-of-non-default-branch/16306 - # schedule: - # - cron: 30 23 * * * - -# [ Note: Workflow/Job Level ENV ] -# Workflow/Job level env doesn't work even though document indicates this feature -# https://github.com/actions/runner/issues/480 -# https://github.community/t/how-to-set-and-access-a-workflow-variable/17335 -# env: -# RELEASE_BRANCH: "" - -jobs: - build_test_upload: - if: github.repository == 'pytorch/data' && startsWith(github.ref_name, 'release/') - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "" - pre_dev_release: true - pytorch_version: "" - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} From d906f23cf75f26b46bc53f825fb0b58f3a398b98 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 11 Jan 2024 18:27:27 -0800 Subject: [PATCH 8/8] Clean up AWS credentials --- .github/workflows/build-wheels-m1.yml | 7 ++++--- .github/workflows/build_wheels_linux.yml | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml index 2fdb5f6e5..d75f0dac0 100644 --- a/.github/workflows/build-wheels-m1.yml +++ b/.github/workflows/build-wheels-m1.yml @@ -13,6 +13,10 @@ on: - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: +permissions: + id-token: write + contents: read + jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main @@ -38,6 +42,3 @@ jobs: runner-type: macos-m1-12 smoke-test-script: test/smoke_test/smoke_test.py trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index b179929b4..b345249ba 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -7,6 +7,10 @@ on: - nightly workflow_dispatch: +permissions: + id-token: write + contents: read + jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main @@ -42,6 +46,3 @@ jobs: package-name: ${{ matrix.package-name }} env-var-script: packaging/env-var-script.txt trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }}