Build GraphScope Wheels on macOS #1212
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build GraphScope Wheels on MacOS | |
| # on: [push, pull_request] | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # The notifications for scheduled workflows are sent to the user who | |
| # last modified the cron syntax in the workflow file. | |
| # Trigger the workflow at 03:00(CST) every day. | |
| - cron: '00 19 * * *' | |
| push: | |
| tags: | |
| - "v*" | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-wheels: | |
| if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') | |
| runs-on: macos-12 | |
| strategy: | |
| matrix: | |
| python-version: ['3.11'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Dependencies | |
| run: | | |
| brew update || true | |
| brew install bash coreutils | |
| alias bash=$(brew --prefix)/bin/bash | |
| export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH | |
| sudo mkdir /opt/graphscope | |
| sudo chown -R $(id -u):$(id -g) /opt/graphscope | |
| # Not sure when the first installation would fail, saying that the lock of python3.10 is taken | |
| # However the second trial would success. | |
| python3 -m pip install click | |
| python3 gsctl.py install-deps dev || true | |
| python3 gsctl.py install-deps dev | |
| echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env | |
| - name: Install JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'zulu' # See 'Supported distributions' for available options | |
| java-version: '11' | |
| - name: Reinstall glog | |
| run: | | |
| # Essential cause glog installed by brew is broken and produce corrupted messages | |
| brew uninstall --ignore-dependencies glog | |
| . ~/.graphscope_env | |
| git clone https://github.com/google/glog.git | |
| cd glog | |
| cmake . -DWITH_TLS=OFF -DWITH_SYMBOLIZE=OFF -DWITH_UNWIND=OFF | |
| make -j | |
| sudo make install | |
| # make delocate-wheel happy | |
| sudo mkdir -p /usr/local/opt/glog/lib | |
| sudo ln -s /usr/local/lib/libglog*.dylib /usr/local/opt/glog/lib/ | |
| - name: Install latest libgrape-lite | |
| if: false | |
| run: | | |
| . ~/.graphscope_env | |
| git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite | |
| cd /tmp/libgrape-lite | |
| cmake . -DCMAKE_INSTALL_PREFIX=/usr/local | |
| make -j4 | |
| sudo make install | |
| - name: Install latest vineyard | |
| if: false | |
| run: | | |
| source ~/.graphscope_env | |
| python3 -m pip install libclang | |
| git clone --single-branch -b v0.13.4 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d | |
| cd /tmp/v6d | |
| git submodule update --init | |
| cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ | |
| -DCMAKE_PREFIX_PATH=/usr/local \ | |
| -DUSE_EXTERNAL_ETCD_LIBS=OFF \ | |
| -DBUILD_SHARED_LIBS=ON \ | |
| -DBUILD_VINEYARD_TESTS=OFF | |
| make -j4 | |
| sudo make install | |
| - name: Build Server Wheel | |
| run: | | |
| # source environment variable | |
| . ~/.graphscope_env | |
| python3 -m pip install libclang | |
| echo ${CC} | |
| # make sure the python's local bin is in PATH (by `pip install --user`) | |
| export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH | |
| # change the version for nightly release | |
| # e.g. 0.15.0 -> 0.15.0a20220808 | |
| time=$(date "+%Y%m%d") | |
| version=$(cat ${GITHUB_WORKSPACE}/VERSION) | |
| if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; | |
| then | |
| echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; | |
| fi | |
| python3 -m pip install numpy pandas "grpcio>=1.49" "grpcio-tools>=1.49" "mypy-protobuf>=3.4.0" wheel | |
| # build graphscope server wheel | |
| cd ${GITHUB_WORKSPACE}/k8s/internal | |
| sudo -E env PATH=$PATH make graphscope-py3-package GRAPHSCOPE_HOME=/usr/local | |
| # package | |
| cd ${GITHUB_WORKSPACE} | |
| tar -zcf graphscope.tar.gz coordinator/dist/wheelhouse/*.whl | |
| - name: Setup tmate session | |
| if: false | |
| uses: mxschmitt/action-tmate@v2 | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheel-macos-${{ github.sha }} | |
| path: graphscope.tar.gz | |
| retention-days: 5 | |
| build-client-wheels: | |
| if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') | |
| runs-on: macos-12 | |
| strategy: | |
| matrix: | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Dependencies | |
| run: | | |
| brew update || true | |
| brew install bash coreutils | |
| alias bash=$(brew --prefix)/bin/bash | |
| sudo mkdir /opt/graphscope | |
| sudo chown -R $(id -u):$(id -g) /opt/graphscope | |
| python3 -m pip install click | |
| python3 gsctl.py install-deps dev || true | |
| python3 gsctl.py install-deps dev | |
| - name: Install latest libgrape-lite | |
| if: false | |
| run: | | |
| . ~/.graphscope_env | |
| git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite | |
| cd /tmp/libgrape-lite | |
| cmake . -DCMAKE_INSTALL_PREFIX=/usr/local | |
| make -j4 | |
| sudo make install | |
| - name: Install latest vineyard | |
| if: false | |
| run: | | |
| . ~/.graphscope_env | |
| python3 -m pip install libclang | |
| git clone --single-branch -b v0.18.2 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d | |
| cd /tmp/v6d | |
| git submodule update --init | |
| cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ | |
| -DCMAKE_PREFIX_PATH=/usr/local \ | |
| -DUSE_EXTERNAL_ETCD_LIBS=OFF \ | |
| -DBUILD_SHARED_LIBS=ON \ | |
| -DBUILD_VINEYARD_TESTS=OFF | |
| make -j4 | |
| sudo make install | |
| - name: Build Client Wheels | |
| run: | | |
| python3 -c "import sys; print(sys.version)" | |
| . ~/.graphscope_env | |
| # make sure the python's local bin is in PATH (by `pip install --user`) | |
| export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH | |
| # change the version for nightly release | |
| # 0.15.0 -> 0.15.0a20220808 | |
| time=$(date "+%Y%m%d") | |
| version=$(cat ${GITHUB_WORKSPACE}/VERSION) | |
| if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then | |
| echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; | |
| fi | |
| # build graphscope client wheel | |
| cd ${GITHUB_WORKSPACE}/k8s/internal | |
| make graphscope-client-py3-package GRAPHSCOPE_HOME=/usr/local | |
| # package | |
| cd ${GITHUB_WORKSPACE} | |
| tar -zcf client.tar.gz python/dist/ | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheel-macos-${{ github.sha }}-${{ matrix.python-version }} | |
| path: client.tar.gz | |
| retention-days: 5 | |
| # Action gh-action-pypi-publish not support non-linux os. | |
| publish-wheels: | |
| if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') | |
| runs-on: ubuntu-20.04 | |
| needs: [build-wheels, build-client-wheels] | |
| strategy: | |
| matrix: | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Prepare Wheels | |
| env: | |
| PYTHON: ${{ matrix.python-version }} | |
| run: | | |
| # move wheels into one floder to upload to PyPI | |
| mkdir ${GITHUB_WORKSPACE}/upload_pypi | |
| cd ${GITHUB_WORKSPACE}/artifacts | |
| tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz | |
| mv ${GITHUB_WORKSPACE}/artifacts/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ | |
| if [ "$PYTHON" == "3.8" ]; then | |
| tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz | |
| mv ${GITHUB_WORKSPACE}/artifacts/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ | |
| fi | |
| # We do this, since failures on test.pypi aren't that bad | |
| - name: Publish to Test PyPI | |
| # the limit of graphscope package in test.pypi is still 100MB, which is not enough. | |
| if: false | |
| uses: pypa/[email protected] | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.TEST_PYPI_PASSWORD }} | |
| repository_url: https://test.pypi.org/legacy/ | |
| packages_dir: upload_pypi/ | |
| - name: Publish distribution to PyPI | |
| uses: pypa/[email protected] | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_PASSWORD }} | |
| packages_dir: upload_pypi/ | |
| python-test: | |
| if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }} | |
| runs-on: macos-12 | |
| needs: [build-wheels, build-client-wheels] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Prepare Environment | |
| run: | | |
| cd ${GITHUB_WORKSPACE}/artifacts | |
| # install graphscope-client | |
| tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz | |
| pushd python/dist/wheelhouse | |
| for f in * ; do python3 -m pip install $f --user || true; done | |
| popd | |
| # install graphscope | |
| tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz | |
| pushd coordinator/dist/wheelhouse | |
| python3 -m pip install ./*.whl --user | |
| popd | |
| # install tensorflow | |
| python3 -m pip install pytest "tensorflow" "pandas" --user | |
| - name: Run Minimum Test | |
| env: | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| export JAVA_HOME=$(/usr/libexec/java_home -v11) | |
| export PATH=$JAVA_HOME/bin:$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH | |
| python3 -c "import sys; print(sys.version)" | |
| git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} | |
| python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest | |
| - name: Setup tmate session | |
| if: false | |
| uses: mxschmitt/action-tmate@v2 |