standalone CUDA is clearly broken. I'm going to move testing to HIP. …

…This is really frustrating.
E3SM-Project · Jan 30, 2024 · b909db1 · b909db1
1 parent 9ba849a
commit b909db1
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 4 deletions.
diff --git a/physics/scream_cxx_p3_shoc/CMakeLists.txt b/physics/scream_cxx_p3_shoc/CMakeLists.txt
@@ -15,9 +15,11 @@ set(EKAT_CONFIGURE_FILE_F90_FILE ON)
 set(Kokkos_ENABLE_SERIAL  ON CACHE BOOL "" FORCE)
 if ("${YAKL_ARCH}" STREQUAL "CUDA")
   set(EAMXX_ENABLE_GPU             ON CACHE BOOL "" FORCE)
-#  set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "" FORCE)
+  set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "" FORCE)
   set(Kokkos_ENABLE_CUDA_LAMBDA    ON CACHE BOOL "" FORCE)
   set(Kokkos_ENABLE_CUDA           ON CACHE BOOL "" FORCE)
+  set($ENV{Kokkos_ARCH_CUDA}       ON CACHE BOOL "" FORCE)
+  set(Kokkos_ARCH_KEPLER35         OFF CACHE BOOL "" FORCE)
 elseif ("${YAKL_ARCH}" STREQUAL "HIP")
   set(EAMXX_ENABLE_GPU                     ON    CACHE BOOL "" FORCE)
   set(Kokkos_ENABLE_HIP                    ON    CACHE BOOL "" FORCE)

diff --git a/standalone/machines/thatchroof/gpu.env b/standalone/machines/thatchroof/gpu.env
@@ -27,4 +27,5 @@ export SCREAM_Fortran_FLAGS="-O3;-ffree-line-length-none"
 export SCREAM_CXX_LIBS_DIR="`pwd`/../../mmf_simplified/build_p3_shoc_cxx"
 export PAM_NLEV=50
 export PAM_SCREAM_USE_CXX="OFF"
+export Kokkos_ARCH_CUDA=Kokkos_ARCH_AMPERE80
 
diff --git a/standalone/machines/thatchroof/gpu_debug.env b/standalone/machines/thatchroof/gpu_debug.env
@@ -20,11 +20,12 @@ export FC=mpif90
 export YAKL_CUDA_FLAGS="-DHAVE_MPI -O0 -g -DYAKL_DEBUG -DPAM_DEBUG -arch sm_86 -ccbin mpic++ -DYAKL_PROFILE  -I`nc-config --includedir`"
 export YAKL_F90_FLAGS="-O0 -g -ffree-line-length-none -DSCREAM_DOUBLE_PRECISION"
 export PAM_LINK_FLAGS="`nc-config --libs`"
-export SCREAM_HOME="/home/$USER/scream"
+export SCREAM_HOME="/home/$USER/E3SM"
 export YAKL_HOME="/home/$USER/YAKL"
 export PAM_SCREAM_CXX_FLAGS="-O3;-DHAVE_MPI;-I/home/$USER/YAKL/external"
 export SCREAM_Fortran_FLAGS="-O3;-ffree-line-length-none"
 export SCREAM_CXX_LIBS_DIR="`pwd`/../../mmf_simplified/build_p3_shoc_cxx"
 export PAM_NLEV=50
-export PAM_SCREAM_USE_CXX="OFF"
+export PAM_SCREAM_USE_CXX="ON"
+export Kokkos_ARCH_CUDA=Kokkos_ARCH_AMPERE80
 
diff --git a/standalone/mmf_simplified/build/cmakescript_pama.sh b/standalone/mmf_simplified/build/cmakescript_pama.sh
@@ -16,7 +16,7 @@ cmake      \
   -DPAM_DYCORE="awfl"                                             \
   -DPAM_MICRO="p3"                                                \
   -DPAM_RAD="none"                                                \
-  -DPAM_SGS="none"                                                \
+  -DPAM_SGS="shoc"                                                \
   -DPAM_RAD="none"                                                \
   -DPAM_NLEV=${PAM_NLEV}                                          \
   -DSCREAM_CXX_LIBS_DIR=${SCREAM_CXX_LIBS_DIR}                    \