From b909db16c2e85162f0da91768fe9fb73da249080 Mon Sep 17 00:00:00 2001 From: Matt Norman Date: Tue, 30 Jan 2024 14:51:57 -0500 Subject: [PATCH] standalone CUDA is clearly broken. I'm going to move testing to HIP. This is really frustrating. --- physics/scream_cxx_p3_shoc/CMakeLists.txt | 4 +++- standalone/machines/thatchroof/gpu.env | 1 + standalone/machines/thatchroof/gpu_debug.env | 5 +++-- standalone/mmf_simplified/build/cmakescript_pama.sh | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/physics/scream_cxx_p3_shoc/CMakeLists.txt b/physics/scream_cxx_p3_shoc/CMakeLists.txt index e95a3955..a507dd09 100644 --- a/physics/scream_cxx_p3_shoc/CMakeLists.txt +++ b/physics/scream_cxx_p3_shoc/CMakeLists.txt @@ -15,9 +15,11 @@ set(EKAT_CONFIGURE_FILE_F90_FILE ON) set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE) if ("${YAKL_ARCH}" STREQUAL "CUDA") set(EAMXX_ENABLE_GPU ON CACHE BOOL "" FORCE) -# set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "" FORCE) + set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "" FORCE) set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "" FORCE) set(Kokkos_ENABLE_CUDA ON CACHE BOOL "" FORCE) + set($ENV{Kokkos_ARCH_CUDA} ON CACHE BOOL "" FORCE) + set(Kokkos_ARCH_KEPLER35 OFF CACHE BOOL "" FORCE) elseif ("${YAKL_ARCH}" STREQUAL "HIP") set(EAMXX_ENABLE_GPU ON CACHE BOOL "" FORCE) set(Kokkos_ENABLE_HIP ON CACHE BOOL "" FORCE) diff --git a/standalone/machines/thatchroof/gpu.env b/standalone/machines/thatchroof/gpu.env index d3812cb7..6d506a70 100644 --- a/standalone/machines/thatchroof/gpu.env +++ b/standalone/machines/thatchroof/gpu.env @@ -27,4 +27,5 @@ export SCREAM_Fortran_FLAGS="-O3;-ffree-line-length-none" export SCREAM_CXX_LIBS_DIR="`pwd`/../../mmf_simplified/build_p3_shoc_cxx" export PAM_NLEV=50 export PAM_SCREAM_USE_CXX="OFF" +export Kokkos_ARCH_CUDA=Kokkos_ARCH_AMPERE80 diff --git a/standalone/machines/thatchroof/gpu_debug.env b/standalone/machines/thatchroof/gpu_debug.env index 1db095f4..253e5d9c 100644 --- a/standalone/machines/thatchroof/gpu_debug.env +++ b/standalone/machines/thatchroof/gpu_debug.env @@ -20,11 +20,12 @@ export FC=mpif90 export YAKL_CUDA_FLAGS="-DHAVE_MPI -O0 -g -DYAKL_DEBUG -DPAM_DEBUG -arch sm_86 -ccbin mpic++ -DYAKL_PROFILE -I`nc-config --includedir`" export YAKL_F90_FLAGS="-O0 -g -ffree-line-length-none -DSCREAM_DOUBLE_PRECISION" export PAM_LINK_FLAGS="`nc-config --libs`" -export SCREAM_HOME="/home/$USER/scream" +export SCREAM_HOME="/home/$USER/E3SM" export YAKL_HOME="/home/$USER/YAKL" export PAM_SCREAM_CXX_FLAGS="-O3;-DHAVE_MPI;-I/home/$USER/YAKL/external" export SCREAM_Fortran_FLAGS="-O3;-ffree-line-length-none" export SCREAM_CXX_LIBS_DIR="`pwd`/../../mmf_simplified/build_p3_shoc_cxx" export PAM_NLEV=50 -export PAM_SCREAM_USE_CXX="OFF" +export PAM_SCREAM_USE_CXX="ON" +export Kokkos_ARCH_CUDA=Kokkos_ARCH_AMPERE80 diff --git a/standalone/mmf_simplified/build/cmakescript_pama.sh b/standalone/mmf_simplified/build/cmakescript_pama.sh index 61b01f84..0de0597d 100755 --- a/standalone/mmf_simplified/build/cmakescript_pama.sh +++ b/standalone/mmf_simplified/build/cmakescript_pama.sh @@ -16,7 +16,7 @@ cmake \ -DPAM_DYCORE="awfl" \ -DPAM_MICRO="p3" \ -DPAM_RAD="none" \ - -DPAM_SGS="none" \ + -DPAM_SGS="shoc" \ -DPAM_RAD="none" \ -DPAM_NLEV=${PAM_NLEV} \ -DSCREAM_CXX_LIBS_DIR=${SCREAM_CXX_LIBS_DIR} \