aimrocks packaging pipeline #175
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: aimrocks packaging pipeline | |
| on: workflow_dispatch | |
| jobs: | |
| uploading-wheels: | |
| needs: | |
| - linux-dist-aarch64 | |
| - linux-dist-x86_64 | |
| - macos-dist | |
| runs-on: ubuntu-latest | |
| name: Uploading wheels | |
| steps: | |
| - name: Setup python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' | |
| architecture: x64 | |
| - name: Install dev dependencies | |
| run: | | |
| python -m pip install twine | |
| - name: Download all the wheels | |
| uses: actions/download-artifact@v3 | |
| - name: Publish wheels | |
| env: | |
| PYPI_PASSWORD: ${{ secrets.pypi_password }} | |
| run: | | |
| python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" artifact/*.whl | |
| linux-dist-aarch64: | |
| runs-on: m1 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| manylinux-version: [ 'manylinux_2_28_aarch64', 'manylinux2014_aarch64', 'manylinux_2_24_aarch64' ] | |
| name: ${{ matrix.manylinux-version }} using Docker on M1 Mac | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v2 | |
| - name: Pull Docker Images | |
| run: | | |
| docker pull quay.io/pypa/${{ matrix.manylinux-version }} | |
| - name: Building dependencies for ${{ matrix.manylinux-version }} | |
| run: | | |
| docker build \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ | |
| --target deps . | |
| - name: Building rocksdb for ${{ matrix.manylinux-version }} | |
| run: | | |
| docker build \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ | |
| --target rocksdb . | |
| - name: Building wheels for ${{ matrix.manylinux-version }} | |
| run: | | |
| docker build \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ | |
| --target wheels . \ | |
| -t aimhubio/aimrocks:${{ matrix.manylinux-version }} | |
| - name: Auditing wheels for ${{ matrix.manylinux-version }} | |
| run: | | |
| mkdir -p manylinux_dist/ && \ | |
| docker run --rm \ | |
| --mount type=bind,source=$PWD/manylinux_dist,target=/opt/aimrocks/manylinux_dist \ | |
| aimhubio/aimrocks:${{ matrix.manylinux-version }} \ | |
| bash -e /opt/aimrocks/docker/audit-wheels.sh | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| path: manylinux_dist/*.whl | |
| linux-dist-x86_64: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| manylinux-version: [ 'manylinux1_x86_64', 'manylinux2010_x86_64', 'manylinux2014_x86_64', 'manylinux_2_24_x86_64' ] | |
| name: ${{ matrix.manylinux-version }} | |
| steps: | |
| - name: Install Docker & images | |
| run: | | |
| apt update && apt install -y docker.io | |
| sudo systemctl enable --now docker | |
| - name: Checkout sources | |
| uses: actions/checkout@v2 | |
| - name: Setup python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' | |
| architecture: x64 | |
| - name: Pull Docker Images | |
| run: | | |
| docker pull quay.io/pypa/${{ matrix.manylinux-version }} | |
| - uses: satackey/[email protected] | |
| # Ignore the failure of a step and avoid terminating the job. | |
| continue-on-error: true | |
| with: | |
| key: aimrocks-cython-manylinux-build-${{ matrix.manylinux-version }}-{hash} | |
| restore-keys: | | |
| aimrocks-cython-manylinux-build-${{ matrix.manylinux-version }}- | |
| - name: Building dependencies for ${{ matrix.manylinux-version }} | |
| run: | | |
| docker build \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ | |
| --target deps . | |
| - name: Building rocksdb for ${{ matrix.manylinux-version }} | |
| run: | | |
| docker build \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ | |
| --target rocksdb . | |
| - name: Building wheels for ${{ matrix.manylinux-version }} | |
| run: | | |
| docker build \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ | |
| --target wheels . \ | |
| -t aimhubio/aimrocks:${{ matrix.manylinux-version }} | |
| - name: Auditing wheels for ${{ matrix.manylinux-version }} | |
| run: | | |
| mkdir -p manylinux_dist/ && \ | |
| docker run --rm \ | |
| --mount type=bind,source=$PWD/manylinux_dist,target=/opt/aimrocks/manylinux_dist \ | |
| aimhubio/aimrocks:${{ matrix.manylinux-version }} \ | |
| bash -e /opt/aimrocks/docker/audit-wheels.sh | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| path: manylinux_dist/*.whl | |
| macos-deps: | |
| runs-on: m1 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| arch: ['arm64', 'x86_64'] | |
| name: Preparing dependencies for ${{ matrix.arch }} Mac build | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: 10.14 | |
| AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} | |
| CMAKE: /opt/homebrew/bin/cmake | |
| steps: | |
| - name: Preparing Build Dir for Dependencies | |
| run: | | |
| mkdir -p $AIM_DEP_DIR | |
| rm -rf $AIM_DEP_DIR/* | |
| - name: Checkout sources | |
| uses: actions/checkout@v2 | |
| - name: Building ZLib | |
| run: | | |
| pwd | |
| ls -lhatr .github/workflows/mac | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-zlib.sh | |
| - name: Building BZip2 | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-bzip2.sh | |
| - name: Building LZ4 | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-lz4.sh | |
| - name: Building Snappy | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-snappy.sh | |
| - name: Building ZSTD | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-zstd.sh | |
| - name: Building RocksDB | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-rocksdb.sh | |
| macos-dist: | |
| runs-on: m1 | |
| needs: macos-deps | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10' , '3.11'] | |
| arch: ['arm64', 'x86_64'] | |
| exclude: | |
| - arch: 'arm64' | |
| python-version: 3.6 | |
| - arch: 'arm64' | |
| python-version: 3.7 | |
| # requirement for custom runners: having cmake and python installed | |
| name: Python ${{ matrix.python-version }} for ${{ matrix.arch }} build | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: 10.14 | |
| AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} | |
| PYTHON: /opt/conda/${{ matrix.arch }}/envs/py${{ matrix.python-version }}/bin/python | |
| steps: | |
| - name: Build and test wheels | |
| run: | | |
| arch -${{matrix.arch}} $PYTHON -m build | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| path: dist/*.whl |