Python package building rework (#2538)

* - Refactored setup.py to remove hacks regarding packaging of wheels for different platforms, improve and cleanup the code - Updated README.txt - Removed old Makefile and build_wheel.sh scripts - Created a new workflow that takes care of building and testing python packages for different platforms/architectures/python versions * Added SPDX headers to the setup.py * - cstest_py: Fixed positional argument since it doesn't accept a `required` flag. It turns to have a mandatory tests folder path - integration_tests.py: Use pathlib to determine the required path - GitHub action: Simplified the tests execution command * GitHub Actions: Run python 3.8 (lowest) and 3.13 (current highest) for native runners only during testings and the rest during tag release * GitHub Action: - Fixed the cibw_build matrix element - Added a step to prepare artifact name * GitHub Action: Added run_tests.py script to run all tests during CI workflow * - Added SPDX headers to the run_tests.py script and to the build-wheels-publish.yml workflow file - Minor fixes to the workflow as pointed out in the PR review - Updated MANIFEST.in to reflect the actual libraries built during python wheel creation process - Use subprocess.run in place of os.system in run_tests.py script * GitHub Action: - Run qemu step only if non-native Linux runner - Added arch:universal2 matrix element for macos-latest runner * Python bindings: Refreshed the list of files needed to be copied for sdist archive * GitHub Action: Commented out arch:x86 matrix elements * GitHub Action: Run qemu step only if non-native Linux runner * GitHub Action: Minor fixes * Python bindings: Added missing .in pattern when collecting src files for sdist archive
capstone-engine · Nov 18, 2024 · 6ad2608 · 6ad2608
1 parent e3bc578
commit 6ad2608
Show file tree

Hide file tree

Showing 13 changed files with 384 additions and 343 deletions.
diff --git a/.github/workflows/build-wheels-publish.yml b/.github/workflows/build-wheels-publish.yml
@@ -0,0 +1,279 @@
+# SPDX-FileCopyrightText: 2024 Antelox <[email protected]>
+# SPDX-License-Identifier: BSD-3
+
+name: Build and publish wheels with cibuildwheel
+
+on:
+  workflow_dispatch:
+    inputs:
+      debugMode:
+        description: 'Debug Mode'
+        required: false
+        default: ''
+        type: choice
+        options:
+          - '0'
+          - '1'
+
+  push:
+    paths-ignore:
+      - ".gitignore"
+      - "CREDITS.TXT"
+      - "ChangeLog"
+      - "README.md"
+      - "docs/**"
+  pull_request:
+
+env:
+  # Enable DEBUG flag either according to the tag release or manual override
+  CAPSTONE_DEBUG: ${{ inputs.debugMode != '' && inputs.debugMode || startsWith(github.ref, 'refs/tags') && '0' || '1' }}
+
+jobs:
+  # job to be executed for every push - testing purpose
+  build_wheels_always:
+    name: Building on ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.cibw_build }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # NOTE: Making this to parallelize and speed up workflow
+          # i686 - manylinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' }
+          # i686 - musllinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' }
+          # x86_64 - manylinux
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' }
+          # x86_64 - musllinux
+          # - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' }
+          # aarch64 - manylinux
+          # - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' }
+          # aarch64 - musllinux
+          # - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' }
+          # macos - x86_64
+          - { os: macos-13, arch: x86_64, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # macos - arm64
+          # - { os: macos-latest, arch: arm64, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # - { os: macos-latest, arch: universal2, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # windows - x86_64
+          - { os: windows-latest, arch: AMD64, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # windows - amd64
+          # - { os: windows-latest, arch: x86, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # windows - arm64
+          # - { os: windows-latest, arch: ARM64, cibw_build: 'cp39* cp313*', cibw_skip: '' }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # https://github.com/actions/upload-artifact/issues/22
+      - name: Prepare a unique name for Artifacts
+        shell: bash
+        run: |
+          # replace not-allowed chars with dash
+          name="cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.cibw_build }}"
+          name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g' | sed -e 's/\-$//')
+          echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
+
+      # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64
+      - uses: actions/setup-python@v5
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        with:
+          python-version: 3.8
+
+      - name: '🛠️ Win MSVC 32 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'x86'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x86
+
+      - name: '🛠️ Win MSVC 64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'AMD64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+
+      - name: '🛠️ Win MSVC ARM64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'ARM64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: amd64_arm64
+
+      - name: '🛠️ Set up QEMU'
+        if: runner.os == 'Linux' && matrix.arch != 'x86_64'
+        uses: docker/setup-qemu-action@v3
+
+      - name: '🚧 cibuildwheel run'
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD_FRONTEND: build
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: ${{ matrix.cibw_skip }}
+          CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_ENVIRONMENT: DEBUG=${{ env.CAPSTONE_DEBUG }}
+          CIBW_ENVIRONMENT_PASS_LINUX: DEBUG
+          # https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64
+          # https://github.com/pypa/cibuildwheel/pull/1169
+          CIBW_TEST_SKIP: "*-win_arm64 cp38-macosx_*:arm64"
+          CIBW_TEST_COMMAND: >
+            python -m pip install {package}/cstest_py &&
+            python {project}/suite/run_tests.py
+        with:
+          package-dir: bindings/python
+          output-dir: wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ./wheelhouse/*.whl
+
+  # To be executed only in case of a tag release
+  build_wheels_all:
+    name: Building on ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.cibw_build }}
+    runs-on: ${{ matrix.os }}
+    if: startsWith(github.ref, 'refs/tags')
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # NOTE: Making this to parallelize and speed up workflow
+          # i686 - manylinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp39-manylinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp310-manylinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp311-manylinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp312-manylinux*', cibw_skip: '' }
+          # i686 - musllinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp39-musllinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp310-musllinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp311-musllinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp312-musllinux*', cibw_skip: '' }
+          # x86_64 - manylinux
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp39-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp310-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp311-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp312-manylinux*', cibw_skip: '' }
+          # x86_64 - musllinux
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp39-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp310-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp311-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp312-musllinux*', cibw_skip: '' }
+          # aarch64 - manylinux
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp39-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp310-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp311-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp312-manylinux*', cibw_skip: '' }
+          # aarch64 - musllinux
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp39-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp310-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp311-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp312-musllinux*', cibw_skip: '' }
+          # macos - x86_64
+          - { os: macos-13, arch: x86_64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          # macos - arm64
+          - { os: macos-latest, arch: arm64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          - { os: macos-latest, arch: universal2, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *39* *313*' }
+          # windows - amd64
+          - { os: windows-latest, arch: AMD64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          # windows - x86
+          # - { os: windows-latest, arch: x86, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          # windows - arm64
+          - { os: windows-latest, arch: ARM64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *39* *313*' }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # https://github.com/actions/upload-artifact/issues/22
+      - name: Prepare a unique name for Artifacts
+        shell: bash
+        run: |
+          # replace not-allowed chars with dash
+          name="cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.cibw_build }}"
+          name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g' | sed -e 's/\-$//')
+          echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
+
+      - name: '🛠️ Win MSVC 32 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'x86'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x86
+
+      - name: '🛠️ Win MSVC 64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'AMD64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+
+      - name: '🛠️ Win MSVC ARM64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'ARM64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: amd64_arm64
+
+      - name: '🛠️ Set up QEMU'
+        if: runner.os == 'Linux' && matrix.arch != 'x86_64'
+        uses: docker/setup-qemu-action@v3
+
+      - name: '🚧 cibuildwheel run'
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD_FRONTEND: build
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: ${{ matrix.cibw_skip }}
+          CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_ENVIRONMENT: DEBUG=${{ env.CAPSTONE_DEBUG }}
+          CIBW_ENVIRONMENT_PASS_LINUX: DEBUG
+          # https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64
+          CIBW_TEST_SKIP: "*-win_arm64"
+          CIBW_TEST_COMMAND: >
+            python -m pip install {package}/cstest_py &&
+            python {project}/suite/run_tests.py
+        with:
+          package-dir: bindings/python
+          output-dir: wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ./wheelhouse/*.whl
+
+  make_sdist:
+    name: Make SDist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Build SDist
+        run: |
+          cd bindings/python
+          python3 -m pip install -U pip build
+          python3 -m build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist-archive
+          path: bindings/python/dist/*.tar.gz
+
+  publish:
+    needs: [ build_wheels_always, build_wheels_all, make_sdist ]
+    environment: pypi
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          path: dist
+
+      - name: Show downloaded artifacts
+        run: ls -laR dist
+
+      - name: '📦 Publish distribution to PyPI'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: ${{ success() }}
+        with:
+          user: __token__
+          password: ${{ secrets.pypi_pass }}
diff --git a/.github/workflows/python-publish-release.yml b/.github/workflows/python-publish-release.yml