From 9076df047bc95f45c74798483d9ae128defaa186 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Wed, 14 Jun 2023 13:16:41 -0400 Subject: [PATCH 1/5] Add e4s HIP build CI --- .github/workflows/CI-e4s.yml | 149 +++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 .github/workflows/CI-e4s.yml diff --git a/.github/workflows/CI-e4s.yml b/.github/workflows/CI-e4s.yml new file mode 100644 index 000000000..347e13d4e --- /dev/null +++ b/.github/workflows/CI-e4s.yml @@ -0,0 +1,149 @@ +name: CI-e4s +on: + push: + branches: + - master + pull_request: + branches: + - master + +concurrency: + group: ${ {github.event_name }}-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{github.event_name == 'pull_request'}} + +jobs: + HIP: + defaults: + run: + shell: bash + strategy: + matrix: + cxx: ['hipcc'] + cmake_build_type: ['Debug', 'Release'] + kokkos_ver: ['3.6.01'] + arborx: ['OFF'] + heffte: ['OFF' ] + hypre: ['OFF'] + liball: ['OFF'] + silo: ['OFF'] + hdf5: ['OFF'] + runs-on: ubuntu-20.04 + container: docker.io/ecpe4s/e4s-base-rocm:23.05 + steps: + - name: Checkout GTest + uses: actions/checkout@v3 + with: + repository: google/googletest + ref: release-1.11.0 + path: gtest + - name: Build gtest + working-directory: gtest + run: | + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/gtest \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} + cmake --build build --parallel 2 + cmake --install build + - name: Checkout kokkos + uses: actions/checkout@v3 + with: + repository: kokkos/kokkos + ref: ${{ matrix.kokkos_ver }} + path: kokkos + - name: Build kokkos + working-directory: kokkos + run: | + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/kokkos \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DKokkos_ENABLE_HIP=ON \ + -DKokkos_ARCH_VEGA908=ON + cmake --build build --parallel 2 + cmake --install build + - name: Checkout arborx + if: ${{ matrix.arborx == 'ArborX' }} + uses: actions/checkout@v3 + with: + repository: arborx/ArborX + ref: v1.2 + path: arborx + - name: Build arborx + if: ${{ matrix.arborx == 'ArborX' }} + working-directory: arborx + run: | + cmake -B build \ + -DCMAKE_PREFIX_PATH=${HOME}/kokkos \ + -DCMAKE_INSTALL_PREFIX=$HOME/arborx \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} + cmake --build build --parallel 2 + cmake --install build + - name: Checkout heffte + if: ${{ matrix.heffte != 'OFF' }} + # actions/checkout doesn't work for external repos yet (actions/checkout#447) + run: | + git clone --depth 1 --branch v2.1.0 https://bitbucket.org/icl/heffte.git heffte + - name: Build heffte + if: ${{ matrix.heffte != 'OFF' }} + working-directory: heffte + run: | + cmake -B build \ + -DCMAKE_CXX_STANDARD="11" \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=$HOME/heffte \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DHeffte_ENABLE_ROCFFT=ON + cmake --build build --parallel 2 + cmake --install build + - name: Checkout ALL + if: ${{ matrix.liball == 'libALL' }} + run: | + git clone --depth 1 --branch v0.9.2 https://gitlab.jsc.fz-juelich.de/SLMS/loadbalancing ALL + - name: Build ALL + if: ${{ matrix.liball == 'libALL' }} + working-directory: ALL + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/ALL + cmake --build build --parallel 2 + cmake --install build + - name: Checkout Cabana + uses: actions/checkout@v3 + - name: Build Cabana + env: + SILO: ${{ matrix.silo }} + HDF5: ${{ matrix.hdf5 }} + run: | + if [[ ${SILO} == 'Silo' ]]; then + cabana_cmake_opts+=( -DCabana_REQUIRE_SILO=ON ) + else + cabana_cmake_opts+=( -DCMAKE_DISABLE_FIND_PACKAGE_SILO=ON ) + fi + if [[ ${HDF5} == 'HDF5' ]]; then + cabana_cmake_opts+=( -DCabana_REQUIRE_HDF5=ON ) + else + cabana_cmake_opts+=( -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON ) + fi + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ + -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ + -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/gtest;$HOME/arborx;$HOME/heffte;$HOME/ALL" \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ + -DCabana_ENABLE_TESTING=ON \ + -DCabana_ENABLE_EXAMPLES=ON \ + -DCabana_ENABLE_PERFORMANCE_TESTING=ON \ + -DCabana_PERFORMANCE_EXPECTED_FLOPS=0 \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DDOXYGEN_WARN_AS_ERROR=FAIL_ON_WARNINGS \ + ${cabana_cmake_opts[@]} + cmake --build build --parallel 2 --verbose + cmake --install build + - name: Test Cabana Export Target + working-directory: example/core_tutorial/01_hello_world + run: | + cmake -B build \ + -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx;$HOME/heffte;$HOME/hypre;$HOME/Cabana" \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} + cmake --build build From 1b7aa9b3721d109ea1a545759510de76ef263d70 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:53:41 -0400 Subject: [PATCH 2/5] Address comments; build all optional deps --- .github/workflows/CI-e4s.yml | 47 ++++++------------------------------ .github/workflows/CI.yml | 3 +-- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/.github/workflows/CI-e4s.yml b/.github/workflows/CI-e4s.yml index 347e13d4e..5b4af5620 100644 --- a/.github/workflows/CI-e4s.yml +++ b/.github/workflows/CI-e4s.yml @@ -20,13 +20,7 @@ jobs: matrix: cxx: ['hipcc'] cmake_build_type: ['Debug', 'Release'] - kokkos_ver: ['3.6.01'] - arborx: ['OFF'] - heffte: ['OFF' ] - hypre: ['OFF'] - liball: ['OFF'] - silo: ['OFF'] - hdf5: ['OFF'] + kokkos_ver: ['4.0.01'] runs-on: ubuntu-20.04 container: docker.io/ecpe4s/e4s-base-rocm:23.05 steps: @@ -41,7 +35,7 @@ jobs: run: | cmake -B build \ -DCMAKE_INSTALL_PREFIX=$HOME/gtest \ - -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} + -DCMAKE_BUILD_TYPE=Release cmake --build build --parallel 2 cmake --install build - name: Checkout kokkos @@ -62,47 +56,42 @@ jobs: cmake --build build --parallel 2 cmake --install build - name: Checkout arborx - if: ${{ matrix.arborx == 'ArborX' }} uses: actions/checkout@v3 with: repository: arborx/ArborX ref: v1.2 path: arborx - name: Build arborx - if: ${{ matrix.arborx == 'ArborX' }} working-directory: arborx run: | + PATH=$PATH:/opt/rocm cmake -B build \ - -DCMAKE_PREFIX_PATH=${HOME}/kokkos \ + -DKokkos_ROOT=${HOME}/kokkos \ -DCMAKE_INSTALL_PREFIX=$HOME/arborx \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} cmake --build build --parallel 2 cmake --install build - name: Checkout heffte - if: ${{ matrix.heffte != 'OFF' }} # actions/checkout doesn't work for external repos yet (actions/checkout#447) run: | git clone --depth 1 --branch v2.1.0 https://bitbucket.org/icl/heffte.git heffte - name: Build heffte - if: ${{ matrix.heffte != 'OFF' }} working-directory: heffte run: | + PATH=$PATH:/opt/rocm cmake -B build \ - -DCMAKE_CXX_STANDARD="11" \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_INSTALL_PREFIX=$HOME/heffte \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ - -DHeffte_ENABLE_ROCFFT=ON + -DHeffte_ENABLE_ROCM=ON cmake --build build --parallel 2 cmake --install build - name: Checkout ALL - if: ${{ matrix.liball == 'libALL' }} run: | git clone --depth 1 --branch v0.9.2 https://gitlab.jsc.fz-juelich.de/SLMS/loadbalancing ALL - name: Build ALL - if: ${{ matrix.liball == 'libALL' }} working-directory: ALL run: | cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/ALL @@ -111,20 +100,8 @@ jobs: - name: Checkout Cabana uses: actions/checkout@v3 - name: Build Cabana - env: - SILO: ${{ matrix.silo }} - HDF5: ${{ matrix.hdf5 }} run: | - if [[ ${SILO} == 'Silo' ]]; then - cabana_cmake_opts+=( -DCabana_REQUIRE_SILO=ON ) - else - cabana_cmake_opts+=( -DCMAKE_DISABLE_FIND_PACKAGE_SILO=ON ) - fi - if [[ ${HDF5} == 'HDF5' ]]; then - cabana_cmake_opts+=( -DCabana_REQUIRE_HDF5=ON ) - else - cabana_cmake_opts+=( -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON ) - fi + PATH=$PATH:/opt/rocm cmake -B build \ -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ @@ -136,14 +113,6 @@ jobs: -DCabana_ENABLE_PERFORMANCE_TESTING=ON \ -DCabana_PERFORMANCE_EXPECTED_FLOPS=0 \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ - -DDOXYGEN_WARN_AS_ERROR=FAIL_ON_WARNINGS \ - ${cabana_cmake_opts[@]} + -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON cmake --build build --parallel 2 --verbose cmake --install build - - name: Test Cabana Export Target - working-directory: example/core_tutorial/01_hello_world - run: | - cmake -B build \ - -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx;$HOME/heffte;$HOME/hypre;$HOME/Cabana" \ - -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} - cmake --build build diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 77dd97fab..33c730912 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -288,7 +288,7 @@ jobs: working-directory: arborx run: | cmake -B build \ - -DCMAKE_PREFIX_PATH=${HOME}/kokkos \ + -DKokkos_ROOT=${HOME}/kokkos \ -DCMAKE_INSTALL_PREFIX=$HOME/arborx \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ @@ -308,7 +308,6 @@ jobs: [[ ${{ matrix.heffte }} == "MKL" ]] && heffte_cmake_opts+=( -DHeffte_ENABLE_MKL=ON ) # FIXME: Remove MKL path below when we update heFFTe cmake -B build \ - -DCMAKE_CXX_STANDARD="11" \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_INSTALL_PREFIX=$HOME/heffte \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ From 26f7e9c20a6709a3a5c5a89f23a21371b645ed9a Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:05:41 -0400 Subject: [PATCH 3/5] fixup: install fftw for e4s CI --- .github/workflows/CI-e4s.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/CI-e4s.yml b/.github/workflows/CI-e4s.yml index 5b4af5620..73eaeb59b 100644 --- a/.github/workflows/CI-e4s.yml +++ b/.github/workflows/CI-e4s.yml @@ -24,6 +24,24 @@ jobs: runs-on: ubuntu-20.04 container: docker.io/ecpe4s/e4s-base-rocm:23.05 steps: + - name: Build fftw + run: | + wget --quiet http://www.fftw.org/fftw-3.3.8.tar.gz --output-document=fftw.tar.gz + mkdir -p fftw + tar -xf fftw.tar.gz -C fftw --strip-components=1 + cd fftw + cmake -B build \ + -D CMAKE_INSTALL_PREFIX=$HOME/fftw \ + -D CMAKE_BUILD_TYPE=Release \ + -D ENABLE_FLOAT=ON + cmake --build build --parallel 2 + cmake --install build + cmake -B build \ + -D CMAKE_INSTALL_PREFIX=$HOME/fftw \ + -D CMAKE_BUILD_TYPE=Release \ + -D ENABLE_FLOAT=OFF + cmake --build build --parallel 2 + cmake --install build - name: Checkout GTest uses: actions/checkout@v3 with: @@ -83,8 +101,10 @@ jobs: cmake -B build \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_INSTALL_PREFIX=$HOME/heffte \ + -DCMAKE_PREFIX_PATH="$HOME/fftw" \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DHeffte_ENABLE_FFTW=ON \ -DHeffte_ENABLE_ROCM=ON cmake --build build --parallel 2 cmake --install build From f616270dee65aa862328fb53886b0f28b0e2f0a6 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:24:54 -0400 Subject: [PATCH 4/5] fixup: add explicit AMD arch for e4s build --- .github/workflows/CI-e4s.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI-e4s.yml b/.github/workflows/CI-e4s.yml index 73eaeb59b..2079af4c7 100644 --- a/.github/workflows/CI-e4s.yml +++ b/.github/workflows/CI-e4s.yml @@ -133,6 +133,7 @@ jobs: -DCabana_ENABLE_PERFORMANCE_TESTING=ON \ -DCabana_PERFORMANCE_EXPECTED_FLOPS=0 \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ - -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON + -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \ + -DGPU_TARGETS="gfx908" cmake --build build --parallel 2 --verbose cmake --install build From afbdb8e970dc1ceab55c725e34957416c8df4bb1 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:25:15 -0400 Subject: [PATCH 5/5] fixup: run e4s with debug only --- .github/workflows/CI-e4s.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-e4s.yml b/.github/workflows/CI-e4s.yml index 2079af4c7..b7b8c3348 100644 --- a/.github/workflows/CI-e4s.yml +++ b/.github/workflows/CI-e4s.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: cxx: ['hipcc'] - cmake_build_type: ['Debug', 'Release'] + cmake_build_type: ['Debug'] kokkos_ver: ['4.0.01'] runs-on: ubuntu-20.04 container: docker.io/ecpe4s/e4s-base-rocm:23.05