build.sh

#!/bin/bash

# Copyright (c) 2019, NVIDIA CORPORATION.

# BlazingSQL build script

# This script is used to build the component(s) in this repo from
# source, and can be called with various options to customize the
# build as needed (see the help output for details)

# Abort script on first error
set -e

NUMARGS=$#
ARGS=$*

# NOTE: ensure all dir changes are relative to the location of this
# script, and that this script resides in the repo dir!
REPODIR=$(cd $(dirname $0); pwd)

VALIDARGS="clean update thirdparty io libengine engine pyblazing algebra disable-aws-s3 disable-google-gs disable-mysql disable-sqlite disable-postgresql -t -v -g -n -h"
HELP="$0 [-v] [-g] [-n] [-h] [-t]
   clean                - remove all existing build artifacts and configuration (start
                          over) Use 'clean thirdparty' to delete thirdparty folder
   update               - update cudf conda packages
   thirdparty           - build the Thirdparty C++ code only
   io                   - build the IO C++ code only
   libengine            - build the engine C++ code only
   engine               - build the engine Python package
   pyblazing            - build the pyblazing Python package
   algebra              - build the algebra Python package
   disable-aws-s3       - flag to disable AWS S3 support for libengine
   disable-google-gs    - flag to disable Google Cloud Storage support for libengine
   disable-mysql        - flag to enable MySQL support for libengine
   disable-sqlite       - flag to enable SQLite support for libengine
   disable-postgresql   - flag to enable PostgreSQL support for libengine
   -t                   - skip tests
   -v                   - verbose build mode
   -g                   - build for debug
   -n                   - no install step
   -h                   - print this text
   default action (no args) is to build and install all code and packages
"

THIRDPARTY_BUILD_DIR=${REPODIR}/thirdparty/aws-cpp/build
IO_BUILD_DIR=${REPODIR}/io/build
LIBENGINE_BUILD_DIR=${REPODIR}/engine/build
ENGINE_BUILD_DIR=${REPODIR}/engine
PYBLAZING_BUILD_DIR=${REPODIR}/pyblazing
ALGEBRA_BUILD_DIR=${REPODIR}/algebra
BUILD_DIRS="${THIRDPARTY_BUILD_DIR} ${IO_BUILD_DIR} ${LIBENGINE_BUILD_DIR}"

# Set defaults for vars modified by flags to this script
VERBOSE=""
QUIET="--quiet"
BUILD_TYPE=RelWithDebInfo
INSTALL_TARGET=install
TESTS="ON"

# Set defaults for vars that may not have been defined externally
#  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
#         CONDA_PREFIX, but there is no fallback from there!
INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}}
PARALLEL_LEVEL=${PARALLEL_LEVEL:="4"}
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$INSTALL_PREFIX/lib:$INSTALL_PREFIX/lib64
export CXXFLAGS="-L$INSTALL_PREFIX/lib"
export CFLAGS=$CXXFLAGS
if [ -z $CUDACXX ]; then
    export CUDACXX=/usr/local/cuda/bin/nvcc
fi

function hasArg {
    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
}

function buildAll {
    ((${NUMARGS} == 0 )) || !(echo " ${ARGS} " | grep -q " [^-]\+ ")
}

if hasArg -h; then
    echo "${HELP}"
    exit 0
fi

# Check for valid usage
if (( ${NUMARGS} != 0 )); then
    for a in ${ARGS}; do
    if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then
        echo "Invalid option: ${a}"
        exit 1
    fi
    done
fi

# Get version number
export GIT_DESCRIBE_TAG=`git describe --tags`
export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
export UCX_PY_VERSION="0.21"

# Process flags
if hasArg -v; then
    VERBOSE=1
    QUIET=""
fi
if hasArg -g; then
    BUILD_TYPE=Debug
fi
if hasArg -n; then
    INSTALL_TARGET=""
fi
if hasArg -t; then
    TESTS="OFF"
fi

# If clean given, run it prior to any other steps
if hasArg clean; then
    # If the dirs to clean are mounted dirs in a container, the
    # contents should be removed but the mounted dirs will remain.
    # The find removes all contents but leaves the dirs, the rmdir
    # attempts to remove the dirs but can fail safely.
    for bd in ${BUILD_DIRS}; do
    if [ -d ${bd} ]; then
        find ${bd} -mindepth 1 -delete
        rmdir ${bd} || true
    fi
    done

    if hasArg thirdparty; then
        rm -rf ${REPODIR}/thirdparty/aws-cpp/
    fi

    exit 0
fi

################################################################################

if buildAll || hasArg io || hasArg libengine || hasArg thirdparty || hasArg update; then
    if hasArg disable-aws-s3; then
        echo "AWS S3 thirdparty won't be downloaded"
    else
        if [ ! -d "${REPODIR}/thirdparty/aws-cpp/" ]; then
            cd ${REPODIR}/thirdparty/
            aws_cpp_version=$(conda list | grep aws-sdk-cpp|tail -n 1|awk '{print $2}')
            echo "aws_cpp_version for aws cpp sdk 3rdparty is: $aws_cpp_version"

            git clone -b $aws_cpp_version --depth=1 https://github.com/aws/aws-sdk-cpp.git ${REPODIR}/thirdparty/aws-cpp/
            mkdir -p ${THIRDPARTY_BUILD_DIR}
            cd ${THIRDPARTY_BUILD_DIR}
            cmake -GNinja \
                -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
                -DCMAKE_INSTALL_LIBDIR=lib \
                -DBUILD_ONLY='s3-encryption' \
                -DENABLE_UNITY_BUILD=on \
                -DENABLE_TESTING=off \
                -DCMAKE_BUILD_TYPE=Release \
                ..
            ninja install
        else
            echo "thirdparty/aws-cpp/ is already installed in ${INSTALL_PREFIX}"
        fi
    fi
fi

################################################################################

if hasArg update; then
    conda install --yes -c rapidsai-nightly -c nvidia -c conda-forge -c defaults librmm=$MINOR_VERSION rmm=$MINOR_VERSION libcudf=$MINOR_VERSION cudf=$MINOR_VERSION dask-cudf=$MINOR_VERSION dask-cuda=$MINOR_VERSION ucx-py=$UCX_PY_VERSION ucx-proc=*=gpu
fi

################################################################################



if buildAll || hasArg io; then
    disabled_aws_s3_flag=""
    if hasArg disable-aws-s3; then
        disabled_aws_s3_flag="-DS3_SUPPORT=OFF"
        echo "AWS S3 support disabled for io"
    fi

    disabled_google_gs_flag=""
    if hasArg disable-google-gs; then
        disabled_google_gs_flag="-DGCS_SUPPORT=OFF"
        echo "Google Cloud Storage support disabled for io"
    fi

    mkdir -p ${IO_BUILD_DIR}
    cd ${IO_BUILD_DIR}
    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
          -DBUILD_TESTING=${TESTS} \
          -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} $disabled_aws_s3_flag $disabled_google_gs_flag ..

    if [[ ${TESTS} == "ON" ]]; then
        make -j${PARALLEL_LEVEL} all
    else
        make -j${PARALLEL_LEVEL} VERBOSE=${VERBOSE}
    fi

    if [[ ${INSTALL_TARGET} != "" ]]; then
        make -j${PARALLEL_LEVEL} install VERBOSE=${VERBOSE}
    fi
fi

if buildAll || hasArg libengine; then
    disabled_aws_s3_flag=""
    if hasArg disable-aws-s3; then
        disabled_aws_s3_flag="-DS3_SUPPORT=OFF"
        echo "AWS S3 support disabled for libengine"
    fi

    disabled_google_gs_flag=""
    if hasArg disable-google-gs; then
        disabled_google_gs_flag="-DGCS_SUPPORT=OFF"
        echo "Google Cloud Storage support disabled for libengine"
    fi

    disable_mysql_flag=""
    if hasArg disable-mysql; then
        disable_mysql_flag="-DMYSQL_SUPPORT=OFF"
        echo "MySQL database support disabled for engine"
    fi

    disable_sqlite_flag=""
    if hasArg disable-sqlite; then
        disable_sqlite_flag="-DSQLITE_SUPPORT=OFF"
        echo "SQLite database support disabled for engine"
    fi

    disable_postgresql_flag=""
    if hasArg disable-postgresql; then
        disable_postgresql_flag="-DPOSTGRESQL_SUPPORT=OFF"
        echo "PostgreSQL database support disabled for engine"
    fi

    echo "Building libengine"
    mkdir -p ${LIBENGINE_BUILD_DIR}
    cd ${LIBENGINE_BUILD_DIR}
    echo "cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DBUILD_TESTING=${TESTS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_EXE_LINKER_FLAGS=$CXXFLAGS .."
    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
          -DBUILD_TESTING=${TESTS} \
          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
          -DCMAKE_EXE_LINKER_FLAGS="$CXXFLAGS" \
          -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
          $disabled_aws_s3_flag \
          $disabled_google_gs_flag \
          $disable_mysql_flag \
          $disable_sqlite_flag \
          $disable_postgresql_flag ..

    echo "Building libengine: make step"
    if [[ ${TESTS} == "ON" ]]; then
        echo "make -j${PARALLEL_LEVEL} all"
        make -j${PARALLEL_LEVEL} all
    else
        echo "make -j${PARALLEL_LEVEL} blazingsql-engine VERBOSE=${VERBOSE}"
        make -j${PARALLEL_LEVEL} blazingsql-engine VERBOSE=${VERBOSE}
    fi

    if [[ ${INSTALL_TARGET} != "" ]]; then
        echo "make -j${PARALLEL_LEVEL} install VERBOSE=${VERBOSE}"
        make -j${PARALLEL_LEVEL} install VERBOSE=${VERBOSE}
        cp libblazingsql-engine.so ${INSTALL_PREFIX}/lib/libblazingsql-engine.so
    fi
fi


if buildAll || hasArg engine; then
    echo "Building engine (cython wrapper)"
    cd ${ENGINE_BUILD_DIR}
    rm -f ./bsql_engine/io/io.h
    rm -f ./bsql_engine/io/io.cpp

    if [[ ${INSTALL_TARGET} != "" ]]; then
        python setup.py build_ext --inplace
        if [ $? != 0 ]; then
            exit 1
        fi
        python setup.py install --single-version-externally-managed --record=record.txt
        if [ $? != 0 ]; then
            exit 1
        fi

        if [[ $CONDA_BUILD -eq 1 ]]; then
            cp `pwd`/cio*.so `pwd`/../../_h_env*/lib/python*/site-packages
            cp -r `pwd`/bsql_engine `pwd`/../../_h_env*/lib/python*/site-packages
        fi
    else
        python setup.py build_ext --inplace --library-dir=${LIBENGINE_BUILD_DIR}
        if [ $? != 0 ]; then
            exit 1
        fi
    fi
fi

if buildAll || hasArg pyblazing; then
    cd ${PYBLAZING_BUILD_DIR}
    if [[ ${INSTALL_TARGET} != "" ]]; then
        python setup.py build_ext --inplace
        if [ $? != 0 ]; then
            exit 1
        fi
        python setup.py install --single-version-externally-managed --record=record.txt
        if [ $? != 0 ]; then
            exit 1
        fi

        if [[ $CONDA_BUILD -eq 1 ]]; then
            cp -r `pwd`/pyblazing `pwd`/../../_h_env*/lib/python*/site-packages
            cp -r `pwd`/blazingsql `pwd`/../../_h_env*/lib/python*/site-packages
        fi
    else
        python setup.py build_ext --inplace
        if [ $? != 0 ]; then
            exit 1
        fi
    fi
fi

if buildAll || hasArg algebra; then
    cd ${ALGEBRA_BUILD_DIR}
    if [[ ${TESTS} == "ON" ]]; then
        mvn clean install -f pom.xml -Dmaven.repo.local=$INSTALL_PREFIX/blazing-protocol-mvn/ $QUIET
    else
        mvn clean install -Dmaven.test.skip=true -f pom.xml -Dmaven.repo.local=$INSTALL_PREFIX/blazing-protocol-mvn/ $QUIET
    fi

    if [[ ${INSTALL_TARGET} != "" ]]; then
        cp blazingdb-calcite-application/target/BlazingCalcite.jar $INSTALL_PREFIX/lib/blazingsql-algebra.jar
        cp blazingdb-calcite-core/target/blazingdb-calcite-core.jar $INSTALL_PREFIX/lib/blazingsql-algebra-core.jar
    fi
fi