[Cylon-RP] Cylon scaling test with radical-pilot

.github/workflows/c-cpp.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Installing UCX
       run: wget https://github.com/openucx/ucx/releases/download/v1.13.1/ucx-1.13.1.tar.gz -P $HOME/ucx && (cd $HOME/ucx && tar xzf ucx-1.13.1.tar.gz) && (cd $HOME/ucx/ucx-1.13.1 && ./contrib/configure-release --prefix=$PWD/install --with-go=no && make -j8 install)
     - name: Build and test
-      run: python3 -m venv ENV && ./build.sh -pyenv $(pwd)/ENV -bpath $(pwd)/build --cpp --test --python --pytest --cmake-flags "-DCYLON_UCX=ON -DUCX_LIBDIR=$HOME/ucx/ucx-1.13.1/install/lib -DUCX_INCLUDEDIR=$HOME/ucx/ucx-1.13.1/install/include"
+      run: python3 -m venv ENV && ./build.sh -pyenv $(pwd)/ENV -bpath $(pwd)/build --cpp --test --cpp --python --pytest --cmake-flags "-DCYLON_UCX=ON -DUCX_LIBDIR=$HOME/ucx/ucx-1.13.1/install/lib -DUCX_INCLUDEDIR=$HOME/ucx/ucx-1.13.1/install/include"
     - name: Build java
       run: ./build.sh -pyenv $(pwd)/ENV -bpath $(pwd)/build --java
 

build.sh

@@ -358,10 +358,10 @@ build_pyarrow() {
 
 build_python_pyarrow() {
   print_line
-  echo "Building Python"
+  echo "Building Pyarrow"
   source "${PYTHON_ENV_PATH}"/bin/activate || exit 1
   read_python_requirements
-  pip install pyarrow==5.0.0 || exit 1
+  pip install pyarrow==9.0.0 || exit 1
 
   ARROW_LIB=$(python3 -c 'import pyarrow as pa; import os; print(os.path.dirname(pa.__file__))') || exit 1
   LD_LIBRARY_PATH="${ARROW_LIB}:${BUILD_PATH}/lib:${LD_LIBRARY_PATH}" || exit 1

conda/environments/cylon.yml

@@ -10,9 +10,9 @@ dependencies:
   - glog
   - openmpi=4.1.3=ha1ae619_105
   - ucx>=1.12.1
-  - cython>=0.29,<0.30
-  - numpy
-  - pandas>=1.0,<1.6.0dev0
+  - cython>=0.29.31,<3
+  - numpy<1.24.4
+  - pandas>=1.0,<2.0.0
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it

conda/environments/cylon_MacOS.yml

@@ -9,9 +9,9 @@ dependencies:
   - pyarrow=9.0.0
   - glog
   - openmpi>=4.1.2
-  - cython>=0.29,<0.30
-  - numpy
-  - pandas>=1.0,<1.6.0dev0
+  - cython>=0.29.31,<3
+  - numpy<1.24.4
+  - pandas>=1.0,<2.0.0
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it

conda/environments/cylon_rivanna_1.yml

@@ -10,7 +10,7 @@ dependencies:
   - glog
   - openmpi=4.1.3=ha1ae619_105
   - ucx>=1.12.1
-  - cython>=0.29,<0.30
+  - cython>=0.29.31,<3
   - numpy
   - pandas>=1.0,<1.6.0dev0
   - fsspec>=0.6.0

conda/environments/cylon_rivanna_2.yml

@@ -10,7 +10,7 @@ dependencies:
   - glog
   #- openmpi=4.1.3=ha1ae619_105
   - ucx>=1.12.1
-  - cython>=0.29,<0.30
+  - cython>=0.29.31,<3
   - numpy
   - pandas>=1.0,<1.6.0dev0
   - fsspec>=0.6.0

conda/environments/gcylon.yml

@@ -9,14 +9,14 @@ dependencies:
   - cmake>=3.23.1,!=3.25.0
   - arrow-cpp=9
   - pyarrow=9.0.0
-  - cython>=0.29,<0.30
+  - cython>=0.29.31,<3
   - cudf=22.12.01
   - cudatoolkit=11.5
   - glog
   - openmpi=4.1.3=ha1ae619_105
   - ucx>=1.12.1
-  - numpy
-  - pandas>=1.0,<1.6.0dev0
+  - numpy<1.24.4
+  - pandas>=1.0,<2.0.0
   - fsspec>=0.6.0
   - setuptools
   # these are for running tests only,

conda/environments/windows.yml

@@ -9,9 +9,9 @@ dependencies:
   - pyarrow=9.0.0
   - glog
   - msmpi
-  - cython>=0.29,<0.30
-  - numpy
-  - pandas>=1.0,<1.6.0dev0
+  - cython>=0.29.31,<3
+  - numpy<1.24.4
+  - pandas>=1.0,<2.0.0
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it

requirements.txt

@@ -1,8 +1,8 @@
 wheel
-numpy
-cython
+cython>=0.29.31,<3
+numpy<1.24.4
+pandas>=1.0,<2.0.0
 fsspec  # required for pandas-parquet
-pandas
 # setuptools==60.0.0  # required for mpi4py
 # mpi4py==3.1.3
 mpi4py

rivanna/README.md

@@ -4,43 +4,57 @@ Arup Sarker ([email protected], [email protected])
 
 
 
-## Intsall instructions
+## Install instructions
 
 Rivanna is an HPC system offerbed by University of Virginia.
-There are two ways you can build cylon on Rivanna.
+This will use custom dependencies of the system gcc, openmpi version.
 
+```shell
 
-### 1. Custom gcc conda install
+git clone https://github.com/cylondata/cylon.git
+cd cylon
 
-This will use custom dependencies of the system gcc, openmpi version and run slurm script.
+module load gcc/9.2.0 openmpi/3.1.6 python/3.7.7 cmake/3.23.3
 
+python -m venv $PWD/cy-rp-env
 
-```shell
-git clone https://github.com/cylondata/cylon.git
-cd cylon
-conda env create -f conda/environments/cylon_rivanna_1.yml
-sbatch rivanna/job_cylon_rivanna_1.slurm
-```
+source $PWD/cy-rp-env/bin/activate
 
-For more details of the dependent libraries and Slurm scripts, Please checkout the following links:
 
-* <https://github.com/cylondata/cylon/tree/main/conda/environments/cylon_rivanna_1.yml>
-* <https://github.com/cylondata/cylon/tree/main/rivanna/job_cylon_rivanna_1.slurm>
+pip install pip -U
+pip install pytest
 
-### 2. Module based conda install.
+export CC=`which gcc`
+export CXX=`which g++`
+CC=gcc MPICC=mpicc pip install --no-binary mpi4py install mpi4py
+pip install -U pytest-mpi
+pip install numpy
+pip install pyarrow==9.0.0
 
-This will build Cylon by using the loaded module of openmpi and gcc.
 
-Create virtual environment
+rm -rf build
+BUILD_PATH=$PWD/build
+export LD_LIBRARY_PATH=$BUILD_PATH/arrow/install/lib64:$BUILD_PATH/glog/install/lib64:$BUILD_PATH/lib64:$BUILD_PATH/lib:$LD_LIBRARY_PATH
+
+./build.sh -pyenv $PWD/cy-rp-env -bpath $(pwd)/build --cpp --python_with_pyarrow --cython --test --cmake-flags "-DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx)"
+
+```
+It will take some time to build. So, grab a coffee!!!
+
+Let's perform a scaling operation with join. Before that, we have to install the dependencies as follow.
 
 ```shell
-git clone https://github.com/cylondata/cylon.git
-cd cylon
-conda env create -f conda/environments/cylon_rivanna_2.yml
-sbatch rivanna/job_cylon_rivanna_2.slurm
+pip install cloudmesh-common
+pip install openssl-python
+python3 -m pip install urllib3==1.26.6
 ```
 
-For more details of the dependent libraries and Slurm scripts, Please checkout below links:
+We will slum script to run the scaling operation.
+
+```shell
+sbatch rivanna/scripts/scaling_job.slurm
+```
+
+For more details of the dependent libraries and Slurm scripts, Please checkout the following links:
 
-<https://github.com/cylondata/cylon/tree/main/conda/environments/cylon_rivanna_2.yml>
-<https://github.com/cylondata/cylon/tree/main/rivanna/job_cylon_rivanna_2.slurm>
+* <https://github.com/cylondata/cylon/tree/main/rivanna/scripts/scaling_job.slurm>

rivanna/rp-scripts/README.md

@@ -0,0 +1,51 @@
+# Running Cylon on Rivanna
+
+Arup Sarker ([email protected], [email protected])
+
+
+
+## Install instructions for Radical Pilot
+
+Rivanna is an HPC system offerbed by University of Virginia.
+This will use custom dependencies of the system gcc, openmpi version.
+Use the same python environment "cylon_rct" for radical-pilot
+
+```shell
+module load gcc/9.2.0 openmpi/3.1.6 python/3.7.7 cmake/3.23.3
+source $HOME/cylon_rct/bin/activate
+pip install radical.pilot
+```
+For checking all dependent library version:
+
+```shell
+radical-stack
+```
+You need to export mongo-db url:
+
+```shell
+export RADICAL_PILOT_DBURL="mongodb:ADD_YOUR_URL"
+```
+Setup is done. Now let's execute scaling with cylon.
+
+```shell
+cd /some_path_to/cylon/rivanna/rp-scripts
+python rp_scaling.py
+```
+
+If you want to make any change in the uva resource file(/some_path_to/radical.pilot/src/radical/pilot/configs) or any other places in the radical pilot source,
+
+```shell
+git clone https://github.com/radical-cybertools/radical.pilot.git
+cd radical.pilot
+```
+For reflecting those change, you need to upgrade radical-pilot by,
+
+```shell
+pip install . --upgrade
+```
+
+To uninstall radical pilot, execute
+
+```shell
+pip uninstall radical.pilot
+```