Skip to content

Commit

Permalink
Create pyarrow build docker image
Browse files Browse the repository at this point in the history
  • Loading branch information
xmatthias committed Oct 22, 2022
1 parent 7678444 commit 272d90a
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 0 deletions.
59 changes: 59 additions & 0 deletions .github/workflows/builder.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
name: Build Pyarrow wheel for ARM

on:
push:
branches:
- main
- ci/*
tags:
pull_request:

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:

build:
runs-on: ubuntu-22.04

steps:
- uses: actions/checkout@v3

# We need docker experimental to pull the ARM image.
- name: Switch docker to experimental
run: |
docker version -f '{{.Server.Experimental}}'
echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker version -f '{{.Server.Experimental}}'
- name: Set up Docker Buildx
id: buildx
uses: crazy-max/[email protected]
with:
buildx-version: latest
qemu-version: latest

- name: Available platforms
run: echo ${{ steps.buildx.outputs.platforms }}

- name: Build Pyarrow wheels
run: |
docker buildx build -f Dockerfile \
--platform linux/arm/v7 \
--load \
-t pyarrow:wheel .
- name: Extract Wheels from image
run: |
docker run --rm -v $(pwd)/assets:/assets \
--platform linux/arm/v7 \
pyarrow:wheel \
cp -r /build/arrow/python/dist /assets/
- name: Archive artifacts
uses: actions/upload-artifact@v3
with:
name: pyarrow_wheels
path: ./assets
83 changes: 83 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
FROM python:3.9.15-slim-bullseye as base

# Setup env
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
# start pyarrow build
ARG ARROW_VERSION=9.0.0

RUN apt-get update \
&& apt-get -y install \
sudo \
autoconf \
bison \
ca-certificates \
cmake \
curl \
flex \
g++ \
gcc \
libatlas-base-dev \
libgfortran5 \
libboost-dev \
libboost-filesystem-dev \
libboost-regex-dev \
libboost-system-dev \
libgflags-dev \
libutf8proc-dev\
libjemalloc-dev \
libssl-dev \
make \
ninja-build \
pkg-config \
rapidjson-dev \
tzdata \
liblz4-dev \
libsnappy-dev \
libzstd-dev \
&& rm -rf /var/lib/apt/lists/*

RUN echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf \
&& python -m pip install -U pip \
&& python -m pip install wheel setuptools numpy pandas==1.4.3 psutil Cython \
&& which python

WORKDIR /build/arrow
RUN curl --silent --show-error --fail --location \
https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz \
| tar --strip-components=1 -xz

ENV ARROW_HOME=/dist
ENV CMAKE_BUILD_PARALLEL_LEVEL=3

WORKDIR /build/arrow/cpp/release
RUN cmake \
-DPYTHON_EXECUTABLE=/usr/local/bin/python3 \
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DCMAKE_INSTALL_LIBDIR=lib \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_ZLIB=ON \
-DARROW_WITH_ZSTD=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_PARQUET=ON \
-DARROW_PYTHON=ON \
-DARROW_BUILD_TESTS=OFF \
-DARROW_PLASMA=ON \
.. \
&& make -j ${CMAKE_BUILD_PARALLEL_LEVEL} \
&& make install

ENV PYARROW_CMAKE_GENERATOR=Ninja
ENV PYARROW_CMAKE_OPTIONS="-DARROW_USE_LD_GOLD=ON"
ENV PYARROW_WITH_PLASMA=1
ENV PYARROW_WITH_PARQUET=1
ENV PYARROW_BUNDLE_ARROW_CPP=1


WORKDIR /build/arrow/python
RUN pip install -r requirements-wheel-build.txt \
&& python3 setup.py build_ext --build-type="release" --bundle-arrow-cpp bdist_wheel \
&& ls -l /build/arrow/python/dist

# COPY --from=pyarrow-deps /build/arrow/python/dist/pyarrow-*.whl .

0 comments on commit 272d90a

Please sign in to comment.