Test GitHub actions

.github/workflows/conda-actions.yml

@@ -0,0 +1,28 @@
+name: Compile and Test on Conda
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Installing Conda Packages
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        $CONDA/bin/conda env update --file conda/envs/gcylon2.yml --name base
+    - name: Test conda
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo "conda-prefix" $CONDA_PREFIX
+
+

README.md

@@ -1,2 +1,5 @@
 # gcylon
 Cylon on GPU
+
+We are in the process of integrating GPU Cylon into the [main Cylon project](https://github.com/cylondata/cylon). 
+The latest version can be accessed from [the pull request](https://github.com/cylondata/cylon/pull/470).

build.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 
 SOURCE_DIR=$(pwd)/cpp
-CPP_BUILD="OFF"
-CYTHON_BUILD="OFF"
 CONDA_CPP_BUILD="OFF"
+CYTHON_BUILD="OFF"
 CONDA_CYTHON_BUILD="OFF"
 JAVA_BUILD="OFF"
 BUILD_ALL="OFF"
@@ -15,7 +14,7 @@ PYTHON_RELEASE="OFF"
 RUN_CPP_TESTS="OFF"
 RUN_PYTHON_TESTS="OFF"
 STYLE_CHECK="OFF"
-INSTALL_PATH=$(pwd)/build
+INSTALL_PATH=
 BUILD_PATH=$(pwd)/build
 CMAKE_FLAGS=""
 
@@ -37,7 +36,7 @@ case $key in
     shift # past value
     ;;
     --cpp)
-    CPP_BUILD="ON"
+    CONDA_CPP_BUILD="ON"
     shift # past argument
     ;;
     --conda_cpp)
@@ -100,9 +99,8 @@ set -- "${POSITIONAL[@]}" # restore positional parameters
 
 echo "PYTHON ENV PATH       = ${PYTHON_ENV_PATH}"
 echo "BUILD PATH            = ${BUILD_PATH}"
-echo "CPP BUILD             = ${CPP_BUILD}"
-echo "CYTHON BUILD          = ${CYTHON_BUILD}"
 echo "CONDA CPP BUILD       = ${CONDA_CPP_BUILD}"
+echo "CYTHON BUILD          = ${CYTHON_BUILD}"
 echo "CONDA CYTHON BUILD    = ${CONDA_CYTHON_BUILD}"
 echo "BUILD ALL             = ${BUILD_ALL}"
 echo "BUILD DEBUG           = ${BUILD_MODE_DEBUG}"
@@ -149,45 +147,28 @@ else
   echo "Install location set to: ${INSTALL_PATH}"
 fi
 
-build_cpp(){
-  print_line
-  echo "Building CPP in ${BUILD_MODE} mode"
-  print_line
-  CPPLINT_CMD=" "
-  if [ "${STYLE_CHECK}" = "ON" ]; then
-    CPPLINT_CMD=${CPPLINT_COMMAND}
-  fi
-  mkdir ${BUILD_PATH}
-  pushd ${BUILD_PATH} || exit 1
-  cmake -DPYCYLON_BUILD=${CYTHON_BUILD} -DPYTHON_EXEC_PATH=${PYTHON_ENV_PATH} \
-      -DCMAKE_BUILD_TYPE=${BUILD_MODE} -DCYLON_WITH_TEST=${RUN_CPP_TESTS} $CPPLINT_CMD $INSTALL_CMD \
-      ${CMAKE_FLAGS} \
-      ${SOURCE_DIR} || exit 1
-  make -j 4 || exit 1
-  printf "Cylon CPP Built Successufully!"
-  popd || exit 1
-  print_line
-}
-
 build_cpp_conda(){
   print_line
   echo "Building Conda CPP in ${BUILD_MODE} mode"
   print_line
 
+  # set install path to conda dir if it is not already set
+  INSTALL_PATH=${INSTALL_PATH:=${PREFIX:=${CONDA_PREFIX}}}
+
   echo "SOURCE_DIR: ${SOURCE_DIR}"
+  BUILD_PATH=$(pwd)/build
   mkdir -p ${BUILD_PATH}
   pushd ${BUILD_PATH} || exit 1
-  # clear all files in the build directory
-  rm -rf ./*
 
   cmake -DPYCYLON_BUILD=${CYTHON_BUILD} -DPYTHON_EXEC_PATH=${PYTHON_ENV_PATH} \
-      -DCMAKE_BUILD_TYPE=${BUILD_MODE} -DCYLON_WITH_TEST=${RUN_CPP_TESTS} $INSTALL_CMD \
+      -DCMAKE_BUILD_TYPE=${BUILD_MODE} -DCYLON_WITH_TEST=${RUN_CPP_TESTS} -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} \
       ${CMAKE_FLAGS} \
       ${SOURCE_DIR} \
       || exit 1
   make -j 4 || exit 1
-  make install || exit 1
   printf "Cylon CPP Built Successfully!"
+  make install || exit 1
+  printf "Cylon Installed Successfully!"
   popd || exit 1
   print_line
 }
@@ -302,10 +283,6 @@ if [ "${BUILD_MODE_RELEASE}" = "ON" ]; then
    	BUILD_MODE=Release	
 fi
 
-if [ "${CPP_BUILD}" = "ON" ]; then
-   	build_cpp
-fi
-
 if [ "${CONDA_CPP_BUILD}" = "ON" ]; then
 	echo "Running conda build"
 	build_cpp_conda

conda/envs/gcylon2.yml

@@ -0,0 +1,20 @@
+name: gcylon_dev
+channels:
+  - rapidsai
+  - nvidia
+  - conda-forge
+  - defaults
+dependencies:
+  - cmake>=3.17
+  - cython>=0.29,<0.30
+  - cudf=21.08
+  - cudatoolkit=11.2
+  - glog>=0.4.0
+  - setuptools>=49.6.0
+  - openmpi>=4.1.1
+  # - ucx>=1.9.0
+  # these are for running tests only,
+  # they are not needed for using pygcylon or compiling it
+  - pytest
+  - pytest-mpi
+  - mpi4py

cpp/CMakeLists.txt

@@ -92,7 +92,6 @@ if("$ENV{CONDA_BUILD}" STREQUAL "1")
 elseif(DEFINED ENV{CONDA_PREFIX})
     set(CMAKE_SYSTEM_PREFIX_PATH "$ENV{CONDA_PREFIX};${CMAKE_SYSTEM_PREFIX_PATH}")
     set(CONDA_INCLUDE_DIRS "$ENV{CONDA_PREFIX}/include")
-    set(CMAKE_INSTALL_INCLUDEDIR "$ENV{BUILD_PREFIX}/include")
     set(CUDF_INCLUDE_DIR "$ENV{CONDA_PREFIX}/include/libcudf/libcudacxx/")
     link_directories($ENV{CONDA_PREFIX}/lib/)
     link_directories($ENV{CONDA_PREFIX}/lib64/)

cpp/src/cylon/CMakeLists.txt

@@ -20,10 +20,10 @@
 # set_source_files_properties(util/murmur3.cpp util/murmur3.hpp PROPERTIES COMPILE_FLAGS -Wno-implicit-fallthrough)
 
 add_library(gcylon SHARED
-        cudf/cudf_a2a.cuh
-        cudf/cudf_a2a.cu
-        cudf/cudf_a2a.hpp
-        cudf/cudf_a2a.cpp
+        cudf/cudf_all_to_all.cuh
+        cudf/cudf_all_to_all.cu
+        cudf/cudf_all_to_all.hpp
+        cudf/cudf_all_to_all.cpp
         cudf/util.hpp
         cudf/gtable.hpp
         cudf/gtable.cpp

cpp/src/cylon/cudf/CMakeLists.txt

@@ -14,5 +14,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-# cylon_install_all_headers("cylon/cudf")

cpp/src/cylon/cudf/cudf_a2a.cpp → cpp/src/cylon/cudf/cudf_all_to_all.cpp

@@ -2,10 +2,10 @@
 // Created by auyar on 3.02.2021.
 //
 
-#include "cudf_a2a.hpp"
+#include "cudf_all_to_all.hpp"
 #include "gtable.hpp"
 #include "util.hpp"
-#include "cudf_a2a.cuh"
+#include "cudf_all_to_all.cuh"
 
 #include <glog/logging.h>
 #include <net/mpi/mpi_communicator.hpp>

cpp/src/cylon/cudf/cudf_a2a.cu → cpp/src/cylon/cudf/cudf_all_to_all.cu

@@ -1,7 +1,7 @@
 //
 // Created by auyar on 3.02.2021.
 //
-#include "cudf_a2a.cuh"
+#include "cudf_all_to_all.cuh"
 
 namespace gcylon {
 

cpp/src/cylon/cudf/gtable.hpp

@@ -4,7 +4,7 @@
 #ifndef CYLON_GTABLE_H
 #define CYLON_GTABLE_H
 
-#include "cudf_a2a.hpp"
+#include "cudf_all_to_all.hpp"
 #include <join/join_config.hpp>
 #include <cudf/io/types.hpp>
 

cpp/src/examples/CMakeLists.txt

@@ -9,7 +9,7 @@ find_package(CUDAToolkit REQUIRED)
 macro(cylon_add_exe EXENAME)
     add_executable(${EXENAME} ${EXENAME}.cpp)
     target_link_libraries(${EXENAME} gcylon)
-    install(TARGETS ${EXENAME} RUNTIME DESTINATION examples)
+    install(TARGETS ${EXENAME} RUNTIME DESTINATION gcylon/examples)
 endmacro(cylon_add_exe)
 
 cylon_add_exe(cudf_test)

cpp/src/examples/a2a_cudf_test.cpp

@@ -2,7 +2,7 @@
 // Created by auyar on 3.02.2021.
 //
 
-#include "cudf/cudf_a2a.hpp"
+#include "cudf/cudf_all_to_all.hpp"
 
 #include <glog/logging.h>
 #include <chrono>

cpp/src/examples/cudf_test.cpp

@@ -1,4 +1,4 @@
-#include "cudf/cudf_a2a.hpp"
+#include "cudf/cudf_all_to_all.hpp"
 #include "cudf/print.hpp"
 
 #include <iostream>

python/examples/shuffle.py

@@ -1,16 +1,15 @@
 import cupy as cp
-import pygcylon as gc
+import pygcylon as gcy
 
-env: gc.CylonEnv = gc.CylonEnv(config=gc.MPIConfig(), distributed=True)
+env: gcy.CylonEnv = gcy.CylonEnv(config=gcy.MPIConfig(), distributed=True)
 print("CylonContext Initialized: My rank: ", env.rank)
 
 start = 100 * env.rank
-df = gc.DataFrame({'first': cp.random.randint(start, start + 10, 10),
+df = gcy.DataFrame({'first': cp.random.randint(start, start + 10, 10),
                    'second': cp.random.randint(start, start + 10, 10)})
 print("initial df from rank: ", env.rank, "\n", df)
 
-hash_columns = [df._cdf._num_indices + 0]
-shuffledDF = gc.frame.shuffle(df.to_cudf(), hash_columns, env)
+shuffledDF = df.shuffle(on="first", ignore_index=True, env=env)
 
 print("shuffled df from rank: ", env.rank, "\n", shuffledDF)
 

python/pygcylon/__init__.py

@@ -16,10 +16,9 @@
 
 from .net.mpi_config import MPIConfig
 from .frame import (
-     CylonEnv,
-     DataFrame,
-     concat,
-     shuffle,
+    CylonEnv,
+    DataFrame,
+    concat,
 )
 from .groupby import (
      GroupByDataFrame

python/pygcylon/data/shuffle.pyx

@@ -25,21 +25,22 @@ from cudf._lib.cpp.table.table_view cimport table_view
 from cudf._lib.cpp.table.table cimport table
 from cudf._lib.table cimport Table
 
-def shuffle(object tbl, hash_columns, context):
+def shuffle(object tbl, hash_columns, ignore_index, context):
     cdef CStatus status
     cdef Table inputTable = tbl
-    cdef table_view inputTview = inputTable.view()
     cdef unique_ptr[table] output
     cdef vector[int] c_hash_columns
     cdef Table outputTable
     cdef shared_ptr[CCylonContext] c_ctx_ptr = pycylon_unwrap_context(context)
+    cdef table_view inputTview = inputTable.data_view() if ignore_index else inputTable.view()
 
     if hash_columns:
         c_hash_columns = hash_columns
 
         status = Shuffle(inputTview, c_hash_columns, c_ctx_ptr, output)
         if status.is_ok():
-            outputTable = Table.from_unique_ptr(move(output), inputTable._column_names, index_names=inputTable._index_names)
+            index_names = None if ignore_index else inputTable._index_names
+            outputTable = Table.from_unique_ptr(move(output), inputTable._column_names, index_names=index_names)
             return outputTable
         else:
             raise ValueError(f"Shuffle operation failed : {status.get_msg().decode()}")