Skip to content

Commit

Permalink
fix install-deps failure on MacOS(intel) platform
Browse files Browse the repository at this point in the history
  • Loading branch information
@lidongze0629
lidongze0629 committed Sep 27, 2024
1 parent a8d0975 commit 6cbbc63
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 116 deletions.
124 changes: 26 additions & 98 deletions .github/workflows/build-graphscope-wheels-macos.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,87 +19,41 @@ concurrency:
jobs:
build-wheels:
if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
runs-on: macos-12
runs-on: macos-13
strategy:
matrix:
python-version: ['3.11']
python-version: ['3.9']

steps:
- uses: actions/checkout@v4
with:
submodules: true

- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Install Dependencies
run: |
brew update || true
brew install bash coreutils
alias bash=$(brew --prefix)/bin/bash
export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
sudo mkdir /opt/graphscope
sudo chown -R $(id -u):$(id -g) /opt/graphscope
# Not sure when the first installation would fail, saying that the lock of python3.10 is taken
# However the second trial would success.
python3 -m pip install click
python3 gsctl.py install-deps dev || true
python3 gsctl.py install-deps dev
echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
- name: Install JDK 11
uses: actions/setup-java@v4
with:
distribution: 'zulu' # See 'Supported distributions' for available options
java-version: '11'

- name: Reinstall glog
- name: Install Dependencies
run: |
# Essential cause glog installed by brew is broken and produce corrupted messages
brew uninstall --ignore-dependencies glog
. ~/.graphscope_env
git clone https://github.com/google/glog.git -b v0.6.0
cd glog
cmake . -DWITH_TLS=OFF -DWITH_SYMBOLIZE=OFF -DWITH_UNWIND=OFF -DWITH_GTEST=OFF -DBUILD_TESTING=OFF
make -j
sudo make install
# make delocate-wheel happy
sudo mkdir -p /usr/local/opt/glog/lib
sudo ln -s /usr/local/lib/libglog*.dylib /usr/local/opt/glog/lib/
python3 -c 'import sys; print(sys.version_info[:])'
export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
- name: Install latest libgrape-lite
if: false
run: |
. ~/.graphscope_env
git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
cd /tmp/libgrape-lite
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
make -j4
sudo make install
brew update || true
brew install bash coreutils
alias bash=$(brew --prefix)/bin/bash
- name: Install latest vineyard
if: false
run: |
source ~/.graphscope_env
python3 -m pip install libclang
git clone --single-branch -b v0.13.4 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
cd /tmp/v6d
git submodule update --init
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_PREFIX_PATH=/usr/local \
-DUSE_EXTERNAL_ETCD_LIBS=OFF \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_VINEYARD_TESTS=OFF
make -j4
sudo make install
python3 -m pip install click
# install the specific vineyard version:
# python3 gsctl.py install-deps dev --v6d-version <branch or tag>
python3 gsctl.py install-deps dev
echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
- name: Build Server Wheel
run: |
Expand All @@ -115,8 +69,7 @@ jobs:
# e.g. 0.15.0 -> 0.15.0a20220808
time=$(date "+%Y%m%d")
version=$(cat ${GITHUB_WORKSPACE}/VERSION)
if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]];
then
if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then
echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
fi
Expand All @@ -143,58 +96,33 @@ jobs:

build-client-wheels:
if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
runs-on: macos-12
runs-on: macos-13
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
python-version: ['3.9', '3.10', '3.11']

steps:
- uses: actions/checkout@v4
with:
submodules: true

- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Install Dependencies
run: |
python3 -c 'import sys; print(sys.version_info[:])'
export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
brew update || true
brew install bash coreutils
alias bash=$(brew --prefix)/bin/bash
sudo mkdir /opt/graphscope
sudo chown -R $(id -u):$(id -g) /opt/graphscope
python3 -m pip install click
python3 gsctl.py install-deps dev || true
python3 gsctl.py install-deps dev
- name: Install latest libgrape-lite
if: false
run: |
. ~/.graphscope_env
git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
cd /tmp/libgrape-lite
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
make -j4
sudo make install
- name: Install latest vineyard
if: false
run: |
. ~/.graphscope_env
python3 -m pip install libclang
git clone --single-branch -b v0.23.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
cd /tmp/v6d
git submodule update --init
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_PREFIX_PATH=/usr/local \
-DUSE_EXTERNAL_ETCD_LIBS=OFF \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_VINEYARD_TESTS=OFF
make -j4
sudo make install
echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
- name: Build Client Wheels
run: |
Expand Down Expand Up @@ -234,7 +162,7 @@ jobs:
needs: [build-wheels, build-client-wheels]
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
python-version: ['3.9', '3.10', '3.11']

steps:
- uses: actions/download-artifact@v4
Expand All @@ -252,7 +180,7 @@ jobs:
tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz
mv ${GITHUB_WORKSPACE}/artifacts/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/
if [ "$PYTHON" == "3.8" ]; then
if [ "$PYTHON" == "3.9" ]; then
tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz
mv ${GITHUB_WORKSPACE}/artifacts/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/
fi
Expand All @@ -277,12 +205,12 @@ jobs:

python-test:
if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }}
runs-on: macos-12
runs-on: macos-13
needs: [build-wheels, build-client-wheels]
strategy:
fail-fast: false
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
python-version: ['3.9', '3.10', '3.11']

steps:
- uses: actions/download-artifact@v4
Expand Down
60 changes: 42 additions & 18 deletions python/graphscope/gsctl/scripts/install_deps.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -574,8 +574,14 @@ install_patchelf() {

# libgrape-lite
install_libgrape_lite() {
if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then
return 0
if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
if [[ -f "${HOMEBREW_PREFIX}/include/grape/grape.h" ]]; then
return 0
fi
else
if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then
return 0
fi
fi
local branch=$1
pushd "${tempdir}" || exit
Expand All @@ -599,7 +605,13 @@ install_vineyard() {
python3 -m pip --no-cache-dir install pip -U --user
python3 -m pip --no-cache-dir install libclang wheel auditwheel --user
auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')")
sed -i 's/p.error/logger.warning/g' ${auditwheel_path}
if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
BUILD_VINEYARD_GRAPH_WITH_GAR="OFF"
sed -i '' 's/p.error/logger.warning/g' ${auditwheel_path}
else
BUILD_VINEYARD_GRAPH_WITH_GAR="ON"
sed -i 's/p.error/logger.warning/g' ${auditwheel_path}
fi
if [[ "${v6d_version}" != "v"* ]]; then
directory="v6d"
file="${directory}-${v6d_version}.tar.gz"
Expand All @@ -619,10 +631,14 @@ install_vineyard() {
-DBUILD_VINEYARD_TESTS=OFF \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_VINEYARD_PYTHON_BINDINGS=ON \
-DBUILD_VINEYARD_GRAPH_WITH_GAR=ON
-DBUILD_VINEYARD_GRAPH_WITH_GAR=${BUILD_VINEYARD_GRAPH_WITH_GAR}
make -j$(nproc)
make install
strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard*
if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
strip "${V6D_PREFIX}"/bin/vineyard*
else
strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard*
fi
pip3 install --no-cache -i https://pypi.org/simple -U "vineyard" "vineyard-io"
cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/
set +e
Expand Down Expand Up @@ -807,24 +823,30 @@ install_analytical_dependencies() {
install_java_and_maven
# install vineyard
if [[ "${no_v6d}" != true ]]; then
if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
brew install vineyard
else
install_vineyard
fi
install_vineyard
# if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
# brew install vineyard
# else
# install_vineyard
# fi
fi
}

install_analytical_java_dependencies() {
# llvm
if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
brew install llvm || true # prevent the `brew link` failure
export CC=${HOMEBREW_PREFIX}/opt/llvm/bin/clang
export CXX=${HOMEBREW_PREFIX}/opt/llvm/bin/clang++
export CPPFLAGS="${CPPFLAGS} -I${HOMEBREW_PREFIX}/opt/llvm/include"
export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}
elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then
${SUDO} apt-get install -y llvm-11-dev lld-11 clang-11
else
if [[ "${OS_VERSION}" -eq "7" ]]; then
${SUDO} yum install -y llvm-toolset-7.0-clang-devel
source /opt/rh/llvm-toolset-7.0/enable
export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/
else
${SUDO} yum install -y llvm-devel clang-devel lld
fi
Expand Down Expand Up @@ -865,7 +887,9 @@ install_interactive_dependencies() {
rustc --version
fi
# opentelemetry
install_opentelemetry
if [[ "${OS_PLATFORM}" != *"Darwin"* ]]; then
install_opentelemetry
fi
}

install_learning_dependencies() {
Expand All @@ -884,9 +908,9 @@ write_env_config() {
} >> "${OUTPUT_ENV_FILE}"
{
if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl"
echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib"
echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib"
echo "export OPENSSL_ROOT_DIR=${HOMEBREW_PREFIX}/opt/openssl"
echo "export OPENSSL_LIBRARIES=${HOMEBREW_PREFIX}/opt/openssl/lib"
echo "export OPENSSL_SSL_LIBRARY=${HOMEBREW_PREFIX}/opt/openssl/lib/libssl.dylib"
elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then
if [[ "${OS_VERSION}" -eq "7" ]]; then
echo "source /opt/rh/devtoolset-8/enable"
Expand Down Expand Up @@ -916,11 +940,11 @@ write_env_config() {
{
if [[ "${for_analytical_java}" == true ]]; then
if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang"
echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++"
echo "export CC=${HOMEBREW_PREFIX}/opt/llvm/bin/clang"
echo "export CXX=${HOMEBREW_PREFIX}/opt/llvm/bin/clang++"
echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}"
echo "export LDFLAGS=\"-L${homebrew_prefix}/opt/llvm/lib\""
echo "export CPPFLAGS=\"-I${homebrew_prefix}/opt/llvm/include\""
echo "export LDFLAGS=\"-L${HOMEBREW_PREFIX}/opt/llvm/lib\""
echo "export CPPFLAGS=\"-I${HOMEBREW_PREFIX}/opt/llvm/include\""
elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then
echo "source /opt/rh/llvm-toolset-7.0/enable || true"
echo "export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/"
Expand Down

0 comments on commit 6cbbc63

Please sign in to comment.