From f3fbd886ec16af9bdacc95aedcbd17ba823b7777 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 20 Aug 2024 19:50:17 +0100 Subject: [PATCH 1/8] Build aarch64 wheels --- .github/workflows/wheel.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index bd3e400c..c7d0fda3 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -8,17 +8,22 @@ concurrency: jobs: build_wheels: - name: Build wheel on ${{ matrix.os }} + name: Build ${{ matrix.archs }} wheel on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: # macos-12 is an intel runner, macos-14 is a arm64 runner os: [ubuntu-latest, windows-latest, macos-12, macos-14] + archs: ["native"] + include: + - os: ubuntu-latest + archs: "aarch64" env: CIBW_TEST_COMMAND: python -c "import numcodecs" CIBW_BUILD: "cp310-* cp311-* cp312-*" CIBW_SKIP: "pp* *-musllinux_* *win32 *_i686 *_s390x" + CIBW_ARCHS: ${{ matrix.archs }} # note: CIBW_ENVIRONMENT is now set in pyproject.toml steps: From 0fdbb510ec1f82356a6832ddfcd1c78a9b0a3fc4 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 20 Aug 2024 20:51:57 +0100 Subject: [PATCH 2/8] Add QEMU setup --- .github/workflows/wheel.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index c7d0fda3..29af800a 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -33,6 +33,10 @@ jobs: - uses: pypa/cibuildwheel@v2.20.0 + - name: Set up QEMU + if: ${{ matrix.archs == 'aarch64' }} + uses: docker/setup-qemu-action@v3 + - uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.os }} From f0bec356a35e08bf2ec1a781330275382beee7f3 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 Aug 2024 08:32:34 +0100 Subject: [PATCH 3/8] Skip sse and avx flags on aarch64 --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 815224aa..9f71d696 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,9 @@ if have_cflags: # respect compiler options set by user pass -elif os.name == 'posix': +elif os.name == 'posix' and os.uname()[4] != 'aarch64': + # These flags aren't recognised by gcc on aarch64 + # (at least when we build the aarch64 wheens on GitHub actions) if disable_sse2: base_compile_args.append('-mno-sse2') elif have_sse2: From 31ee5d5d57cdc26465f6957c89dc53cef15a20e1 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 23 Aug 2024 15:49:50 +0100 Subject: [PATCH 4/8] Explicitly specify architectures --- .github/workflows/wheel.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 29af800a..7ae139c6 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -8,22 +8,21 @@ concurrency: jobs: build_wheels: - name: Build ${{ matrix.archs }} wheel on ${{ matrix.os }} + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: # macos-12 is an intel runner, macos-14 is a arm64 runner os: [ubuntu-latest, windows-latest, macos-12, macos-14] - archs: ["native"] - include: - - os: ubuntu-latest - archs: "aarch64" + build_all: + - ${{ (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')) || contains(github.event.pull_request.labels.*.name, 'test all wheels')}} env: CIBW_TEST_COMMAND: python -c "import numcodecs" CIBW_BUILD: "cp310-* cp311-* cp312-*" - CIBW_SKIP: "pp* *-musllinux_* *win32 *_i686 *_s390x" - CIBW_ARCHS: ${{ matrix.archs }} + CIBW_ARCHS_MACOS: native + CIBW_ARCHS_WINDOWS: native + CIBW_ARCHS_LINUX: "x86_64 aarch64" # note: CIBW_ENVIRONMENT is now set in pyproject.toml steps: @@ -31,12 +30,17 @@ jobs: with: submodules: true - - uses: pypa/cibuildwheel@v2.20.0 + - name: Restrict number of wheel builds on PRs + if: ${{ !matrix.build_all }} + run: | + echo "CIBW_BUILD=cp310-win_amd64 cp311-manylinux_x86_64 cp312-macosx_x86_64 cp312-macosx_arm64" >> $GITHUB_ENV - name: Set up QEMU - if: ${{ matrix.archs == 'aarch64' }} + if: ${{ matrix.os == 'ubuntu-latest' }} uses: docker/setup-qemu-action@v3 + - uses: pypa/cibuildwheel@v2.20.0 + - uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.os }} From 22abbdc74bd889c5e156371c9f62a02fa50ff013 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 23 Aug 2024 15:57:26 +0100 Subject: [PATCH 5/8] pre-commit fixes --- notebooks/benchmark_vlen.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/notebooks/benchmark_vlen.ipynb b/notebooks/benchmark_vlen.ipynb index d85c1520..8878a2cf 100644 --- a/notebooks/benchmark_vlen.ipynb +++ b/notebooks/benchmark_vlen.ipynb @@ -55,10 +55,10 @@ " enc = codec.encode(a)\n", " print('decode')\n", " %timeit codec.decode(enc)\n", - " print('size : {:,}'.format(len(enc)))\n", - " print('size (zstd 1): {:,}'.format(len(zstd1.encode(enc))))\n", - " print('size (zstd 5): {:,}'.format(len(zstd5.encode(enc))))\n", - " print('size (zstd 9): {:,}'.format(len(zstd9.encode(enc))))" + " print(f'size : {len(enc):,}')\n", + " print(f'size (zstd 1): {len(zstd1.encode(enc)):,}')\n", + " print(f'size (zstd 5): {len(zstd5.encode(enc)):,}')\n", + " print(f'size (zstd 9): {len(zstd9.encode(enc)):,}')" ] }, { From ca429435ce6d680c2ab29ae3560cf8b9cd0f51fa Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 23 Aug 2024 16:09:31 +0100 Subject: [PATCH 6/8] Fix build_all conditional --- .github/workflows/wheel.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 7ae139c6..502f906a 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -15,8 +15,16 @@ jobs: matrix: # macos-12 is an intel runner, macos-14 is a arm64 runner os: [ubuntu-latest, windows-latest, macos-12, macos-14] - build_all: - - ${{ (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')) || contains(github.event.pull_request.labels.*.name, 'test all wheels')}} + # Build all wheels on either: + # - tagged release + # - pull request opened with "test all wheels" label + # - pull requests when "test all wheels" label is added + build_all: | + ${{ + (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')) + || contains(github.event.pull_request.labels.*.name, 'test all wheels') + || (github.event.action == 'labeled' && github.event.label.name == 'test all wheels') + }} env: CIBW_TEST_COMMAND: python -c "import numcodecs" CIBW_BUILD: "cp310-* cp311-* cp312-*" From 8c461d83c3a9c0befbc1ba33daa1ab33c8273d73 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 23 Aug 2024 16:25:24 +0100 Subject: [PATCH 7/8] Use environment variable for build_all --- .github/workflows/wheel.yaml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 502f906a..638f571a 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -6,6 +6,19 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + +env: + # Build all wheels on either: + # - tagged release + # - pull request opened with "test all wheels" label + # - pull requests when "test all wheels" label is added + BUILD_ALL: | + ${{ + (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')) + || contains(github.event.pull_request.labels.*.name, 'test all wheels') + || (github.event.action == 'labeled' && github.event.label.name == 'test all wheels') + }} + jobs: build_wheels: name: Build wheels on ${{ matrix.os }} @@ -15,16 +28,6 @@ jobs: matrix: # macos-12 is an intel runner, macos-14 is a arm64 runner os: [ubuntu-latest, windows-latest, macos-12, macos-14] - # Build all wheels on either: - # - tagged release - # - pull request opened with "test all wheels" label - # - pull requests when "test all wheels" label is added - build_all: | - ${{ - (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')) - || contains(github.event.pull_request.labels.*.name, 'test all wheels') - || (github.event.action == 'labeled' && github.event.label.name == 'test all wheels') - }} env: CIBW_TEST_COMMAND: python -c "import numcodecs" CIBW_BUILD: "cp310-* cp311-* cp312-*" @@ -39,7 +42,7 @@ jobs: submodules: true - name: Restrict number of wheel builds on PRs - if: ${{ !matrix.build_all }} + if: ${{ !env.BUILD_ALL }} run: | echo "CIBW_BUILD=cp310-win_amd64 cp311-manylinux_x86_64 cp312-macosx_x86_64 cp312-macosx_arm64" >> $GITHUB_ENV From 689c3e4c2f75e9e771620879abe239ec66b07171 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 23 Aug 2024 19:38:48 +0100 Subject: [PATCH 8/8] Add changelog entry --- docs/release.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index 1d8a2d41..08a13920 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,7 +13,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ - +* Build aarch64 wheels on linuxL + By :user:`David Stansby `, :issue:`531` Fix ~~~ @@ -27,6 +28,13 @@ Maintenance * Remove pin on Sphinx By :user:`Elliott Sales de Andrade `, :issue:`552`. +* Restrict the number of wheels built on pull requests. + By :user:`David Stansby `, :issue:`531` + +* Add an option to build all wheels on pull requests by adding + the 'build all wheels' label. + By :user:`David Stansby `, :issue:`531` + .. _release_0.13.0: