Add option to build ESMX application with pf #29
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ParFlow CI Test | |
| on: [push, pull_request] | |
| jobs: | |
| build: | |
| name: ${{ matrix.config.name }} | |
| runs-on: ${{ matrix.config.os }} | |
| strategy: | |
| # fail-fast: true | |
| matrix: | |
| config: | |
| - { | |
| name: "Ubuntu 20.04", | |
| os: ubuntu-20.04, | |
| cudaos: 'ubuntu2004', | |
| python: "false", | |
| backend: "none", | |
| amps_layer: mpi1 | |
| } | |
| - { | |
| name: "Ubuntu 20.04 OASIS3-MCT Build", | |
| os: ubuntu-20.04, | |
| cudaos: 'ubuntu2004', | |
| python: "false", | |
| backend: "omp", | |
| amps_layer: oas3 | |
| } | |
| - { | |
| name: "Ubuntu 18.04", | |
| os: ubuntu-18.04, | |
| cudaos: 'ubuntu1804', | |
| python: "false", | |
| backend: "none", | |
| amps_layer: mpi1 | |
| } | |
| - { | |
| name: "Ubuntu 18.04 Python", | |
| os: ubuntu-18.04, | |
| cudaos: 'ubuntu1804', | |
| python: "true", | |
| backend: "none", | |
| amps_layer: mpi1 | |
| } | |
| - { | |
| name: "Ubuntu 18.04 OMP", | |
| os: ubuntu-18.04, | |
| cudaos: 'ubuntu1804', | |
| python: "false", | |
| backend: "omp", | |
| amps_layer: mpi1 | |
| } | |
| - { | |
| name: "Ubuntu 18.04 CUDA Build", | |
| os: ubuntu-18.04, | |
| cc: "gcc", cxx: "g++", | |
| python: "false", | |
| backend: "cuda", | |
| cudaos: 'ubuntu1804', | |
| amps_layer: mpi1 | |
| } | |
| - { | |
| name: "Ubuntu 18.04 Kokkos Build", | |
| os: ubuntu-18.04, | |
| cc: "gcc", cxx: "g++", | |
| python: "false", | |
| backend: "kokkos", | |
| cudaos: 'ubuntu1804', | |
| amps_layer: mpi1 | |
| } | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Package Install 20.04 | |
| if: matrix.config.os == 'ubuntu-20.04' | |
| run: | | |
| sudo apt-get -qq update | |
| sudo apt -qq install gfortran libhdf5-openmpi-dev libhdf5-openmpi-103 hdf5-helpers tcl-dev tk-dev | |
| - name: Package Install 18.04 CUDA | |
| if: (matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos') && matrix.config.os == 'ubuntu-18.04' | |
| run: | | |
| sudo apt-get -qq update | |
| sudo apt-get -qq install -y software-properties-common | |
| sudo add-apt-repository -y main | |
| sudo add-apt-repository -y universe | |
| sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 78BD65473CB3BD13 | |
| sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 6B05F25D762E3157 | |
| - name: Package Install 18.04 | |
| if: matrix.config.os == 'ubuntu-18.04' | |
| run: | | |
| sudo apt-get -qq update | |
| sudo apt -qq install gfortran libhdf5-openmpi-dev libhdf5-openmpi-100 hdf5-helpers tcl-dev tk-dev | |
| - name: Python Package Install | |
| if: matrix.config.python == 'true' | |
| run: | | |
| sudo apt-get -qq update | |
| sudo apt -qq install python3 python3-venv | |
| - name: NetCDF Package Install | |
| if: matrix.config.amps_layer == 'oas3' | |
| run: | | |
| sudo apt-get -qq -y update | |
| sudo apt -qq -y install libnetcdf-dev libnetcdff-dev | |
| # Cache dependencies so don't have to rebuild on each test. | |
| # Can flush caches by resetting the CACHE_VERSION secret on GitHub | |
| # settings for the project (using date-timestamp for secret). | |
| - name: Cache dependencies | |
| uses: actions/cache@v2 | |
| id: cache-parflow-dependencies | |
| env: | |
| cache-name: cache-parflow-dependencies | |
| cache-parflow-hit: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| with: | |
| path: "~/depend" | |
| key: cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }} | |
| - name: Directory Setup | |
| run: | | |
| mkdir -p $HOME/install | |
| echo "PARFLOW_DIR=$HOME/install" >> $GITHUB_ENV | |
| echo "Setting PARFLOW_DIR=$HOME/install" | |
| mkdir -p $HOME/depend/{include,lib} | |
| echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV | |
| echo "Setting PARFLOW_DEP_DIR=$HOME/depend" | |
| echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV | |
| echo "$HOME/depend/bin" >> $GITHUB_PATH | |
| export LD_LIBRARY_PATH=${PARFLOW_DEP_DIR}/lib64:${PARFLOW_DEP_DIR}/lib:${LD_LIBRARY_PATH} | |
| echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV | |
| - name: CMake Install | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| echo "Installing" | |
| cd ~/depend | |
| echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}" > ~/depend/cache-key | |
| wget -nv --no-check-certificate http://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.tar.gz | |
| tar -xf cmake-3.14.0-Linux-x86_64.tar.gz | |
| ~/depend/cmake-3.14.0-Linux-x86_64/bin/cmake --version | |
| fi | |
| echo "$HOME/depend/cmake-3.14.0-Linux-x86_64/bin" >> $GITHUB_PATH | |
| - name: NVidia Driver Install | |
| if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos' | |
| run: | | |
| sudo add-apt-repository -y ppa:graphics-drivers | |
| sudo apt-get update -qq | |
| sudo apt-get install -y nvidia-driver-440 | |
| - name: CUDA Install | |
| if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos' | |
| run: | | |
| CUDA_SHORT=10.1 | |
| CUDA_VER=10.1.105-1 | |
| wget -nv http://developer.download.nvidia.com/compute/cuda/repos/${{ matrix.config.cudaos }}/x86_64/cuda-repo-${{ matrix.config.cudaos }}_${CUDA_VER}_amd64.deb | |
| sudo dpkg -i cuda-repo-${{ matrix.config.cudaos }}_${CUDA_VER}_amd64.deb | |
| sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/${{ matrix.config.cudaos }}/x86_64/3bf863cc.pub | |
| sudo apt update -qq | |
| sudo apt install -qq -y cuda-core-${CUDA_SHORT/./-} cuda-cudart-dev-${CUDA_SHORT/./-} cuda-nvtx-${CUDA_SHORT/./-} | |
| export CUDA_HOME=/usr/local/cuda-${CUDA_SHORT} | |
| echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV | |
| export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} | |
| echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV | |
| export LIBRARY_PATH=${CUDA_HOME}/lib64/stubs | |
| echo "LIBRARY_PATH=${LIBRARY_PATH}" >> $GITHUB_ENV | |
| echo "${CUDA_HOME}/bin" >> $GITHUB_PATH | |
| - name: UCX | |
| if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos' | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| sudo apt install -qq -y autoconf automake libtool | |
| wget -nv https://github.com/openucx/ucx/releases/download/v1.8.0-rc1/ucx-1.8.0.tar.gz | |
| tar -xf ucx-1.8.0.tar.gz | |
| cd ucx-1.8.0 | |
| ./contrib/configure-release --with-cuda=$CUDA_HOME --with-java=no --disable-numa --prefix=$PARFLOW_DEP_DIR | |
| make -j 2 install | |
| fi | |
| - name: RMM Install | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos' | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| git clone -b branch-0.10 --single-branch --recurse-submodules https://github.com/rapidsai/rmm.git | |
| cd rmm | |
| cmake . -DCMAKE_INSTALL_PREFIX=$PARFLOW_DEP_DIR | |
| make -j 2 install | |
| fi | |
| echo "RMM_FLAGS=-DRMM_ROOT=$PARFLOW_DEP_DIR" >> $GITHUB_ENV | |
| - name: Kokkos Install | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| if: matrix.config.backend == 'kokkos' | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| wget -nv https://github.com/kokkos/kokkos/archive/refs/tags/3.3.01.tar.gz | |
| tar -xf 3.3.01.tar.gz | |
| cd kokkos-3.3.01 | |
| mkdir build && cd build | |
| cmake .. -DCMAKE_CXX_COMPILER=$(pwd)/../bin/nvcc_wrapper -DCMAKE_INSTALL_PREFIX=$PARFLOW_DEP_DIR -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_CUDA_UVM=On -DKokkos_ARCH_VOLTA70=On | |
| make -j 2 install | |
| fi | |
| echo "KOKKOS_FLAGS=-DKOKKOS_ROOT=$PARFLOW_DEP_DIR" >> $GITHUB_ENV | |
| - name: OpenMPI Install for CUDA | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos' | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| wget -nv https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz | |
| tar -xf openmpi-4.0.3.tar.gz | |
| cd openmpi-4.0.3 | |
| ./configure --with-cuda=$CUDA_HOME --with-ucx=$PARFLOW_DEP_DIR --prefix=$PARFLOW_DEP_DIR | |
| sudo make -j 2 install | |
| cd .. | |
| fi | |
| sudo ldconfig | |
| - name: OASIS3-MCT Install | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| if: matrix.config.amps_layer == 'oas3' | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| git clone -b OASIS3-MCT_5.0 https://gitlab.com/cerfacs/oasis3-mct.git | |
| cd oasis3-mct | |
| git checkout dd00d6c72b13bf1 | |
| export OASIS_ROOT=$(pwd) | |
| cd util/make_dir | |
| echo 'include $(GITHUB_WORKSPACE)/misc/build_scripts/build.oasis3.ubuntu20.04' > make.inc | |
| make realclean -f TopMakefileOasis3 | |
| make -f TopMakefileOasis3 | |
| mv -v ${OASIS_ROOT}/install/include/* ${PARFLOW_DEP_DIR}/include/ | |
| mv -v ${OASIS_ROOT}/install/lib/* ${PARFLOW_DEP_DIR}/lib/ | |
| fi | |
| - name: SILO Install | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| # wget -nv https://wci.llnl.gov/sites/wci/files/2021-01/silo-4.10.2.tgz | |
| # Use mirror for reliablity | |
| wget -nv https://raw.githubusercontent.com/parflow/parflow-dependencies/master/silo-4.10.2.tar.gz | |
| tar -xf silo-4.10.2.tar.gz | |
| cd silo-4.10.2 | |
| ./configure --disable-silex --disable-hzip --disable-fpzip --prefix=$PARFLOW_DEP_DIR | |
| make -j 2 install | |
| fi | |
| - name: HYPRE Install | |
| env: | |
| CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}} | |
| run: | | |
| if [[ "$CACHE_HIT" != 'true' ]]; then | |
| wget -nv https://github.com/hypre-space/hypre/archive/v2.18.2.tar.gz | |
| tar -xf v2.18.2.tar.gz | |
| cd hypre-2.18.2/src | |
| ./configure --prefix=$PARFLOW_DEP_DIR | |
| make -j 2 install | |
| fi | |
| - name: ParFlow CMake Configure | |
| run: | | |
| cat ~/depend/cache-key | |
| printenv PATH | |
| which mpicc | |
| which cmake | |
| export OMPI_MCA_rmaps_base_oversubscribe=1 | |
| export OMP_NUM_THREADS=1 | |
| if [[ "${{ matrix.config.amps_layer }}" == "oas3" ]]; then HAVE_CLM="OFF"; else HAVE_CLM="ON"; fi | |
| echo "HAVE_CLM=${HAVE_CLM}" | |
| CC=mpicc CXX=mpicxx F77=mpif77 FC=mpif90 cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-Wall -Werror -Wno-unused-result -Wno-unused-function" -DPARFLOW_ENABLE_TIMING=TRUE -DPARFLOW_AMPS_LAYER=${{ matrix.config.amps_layer }} -DMPIEXEC_POSTFLAGS='--oversubscribe' -DPARFLOW_ACCELERATOR_BACKEND=${{ matrix.config.backend }} -DPARFLOW_AMPS_SEQUENTIAL_IO=true -DPARFLOW_HAVE_CLM=${HAVE_CLM} -DHYPRE_ROOT=$PARFLOW_DEP_DIR -DOAS3_ROOT=$PARFLOW_DEP_DIR -DSILO_ROOT=$PARFLOW_DEP_DIR -DPARFLOW_ENABLE_PYTHON=${{ matrix.config.python }} -DPARFLOW_PYTHON_VIRTUAL_ENV=${{ matrix.config.python }} $KOKKOS_FLAGS $RMM_FLAGS -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR | |
| - name: ParFlow CMake Build | |
| run: (cd build; make -j 2 install) | |
| # Can't test with GPU since have no GPU hardware on testing nodes, GPU test is build only | |
| - name: ParFlow CTest | |
| if: matrix.config.backend != 'cuda' && matrix.config.backend != 'kokkos' | |
| run: | | |
| export OMPI_MCA_rmaps_base_oversubscribe=1 | |
| export OMP_NUM_THREADS=1 | |
| (cd build; ctest --output-on-failure) | |