From 1fc37cb25629e8cea3646b6f96fcefda5beeec98 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sat, 11 Jan 2025 19:14:42 -0700
Subject: [PATCH 1/3] add new CLUBB code for GPU compilation

---
 machines/Depends.nvhpc            | 20 ++------------------
 machines/cmake_macros/nvhpc.cmake |  6 ++++++
 2 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
index 5570ef17..af77b187 100644
--- a/machines/Depends.nvhpc
+++ b/machines/Depends.nvhpc
@@ -13,20 +13,6 @@ REDUCED_OPT_OBJS=\
 SatellitePhenologyMod.o \
 esm.o
 
-REDUCED_ERRORCHKS_IN_DEBUG=\
-fvm_consistent_se_cslam.o \
-clm_instMod.o \
-mpp.o \
-horiz_interp_conserve.o \
-data_override.o \
-MOM_io_infra.o \
-dynColumnStateUpdaterMod.o \
-prim_advection_mod.o \
-MARBL_forcing_mod.o \
-MOM_internal_tide_input.o \
-MOM_ice_shelf.o \
-mom_surface_forcing_nuopc.o
-
 REDUCED_PRECISION_OBJS=\
 shr_wv_sat_mod.o
 
@@ -75,6 +61,7 @@ mo_rte_solver_kernels.o \
 mo_optical_props_kernels.o
 
 CLUBB_OBJS=\
+clubb_intr.o\
 adg1_adg2_3d_luhar_pdf.o\
 advance_clubb_core_module.o\
 advance_helper_module.o\
@@ -168,12 +155,9 @@ ifeq ($(DEBUG),FALSE)
 #  $(SHR_RANDNUM_C_OBJS): %.o: %.c
 #	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
   $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(OPENACC_GPU_FLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt -noacc $<
   $(RRTMGP_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
   $(CLUBB_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
-else
-  $(REDUCED_ERRORCHKS_IN_DEBUG): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnobounds -Ktrap=none -Knoieee $<
 endif
diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake
index 39c91fce..33d3664d 100644
--- a/machines/cmake_macros/nvhpc.cmake
+++ b/machines/cmake_macros/nvhpc.cmake
@@ -66,4 +66,10 @@ if (GPU_TYPE STREQUAL a100)
   if (OPENMP_GPU_OFFLOAD)
     string(APPEND OPENMP_GPU_FLAGS " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ")
   endif()
+endif()
+if (OPENACC_GPU_FLAGS)
+  string(APPEND LDFLAGS " ${OPENACC_GPU_FLAGS}")
+endif()
+if (OPENMP_GPU_FLAGS)
+  string(APPEND LDFLAGS " ${OPENMP_GPU_FLAGS}")
 endif()
\ No newline at end of file

From ac33c4254451e7133ab414eba66a0e1c06fe3f24 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sat, 11 Jan 2025 19:17:29 -0700
Subject: [PATCH 2/3] add back some changes

---
 machines/Depends.nvhpc | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
index af77b187..bb15700a 100644
--- a/machines/Depends.nvhpc
+++ b/machines/Depends.nvhpc
@@ -9,6 +9,20 @@ derivative_mod.o \
 bndry_mod.o \
 prim_advance_mod.o
 
+REDUCED_ERRORCHKS_IN_DEBUG=\
+fvm_consistent_se_cslam.o \
+clm_instMod.o \
+mpp.o \
+horiz_interp_conserve.o \
+data_override.o \
+MOM_io_infra.o \
+dynColumnStateUpdaterMod.o \
+prim_advection_mod.o \
+MARBL_forcing_mod.o \
+MOM_internal_tide_input.o \
+MOM_ice_shelf.o \
+mom_surface_forcing_nuopc.o
+
 REDUCED_OPT_OBJS=\
 SatellitePhenologyMod.o \
 esm.o
@@ -155,9 +169,12 @@ ifeq ($(DEBUG),FALSE)
 #  $(SHR_RANDNUM_C_OBJS): %.o: %.c
 #	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
   $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt -noacc $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(OPENACC_GPU_FLAGS) $<
   $(RRTMGP_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
   $(CLUBB_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
+else
+  $(REDUCED_ERRORCHKS_IN_DEBUG): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnobounds -Ktrap=none -Knoieee $<
 endif

From 0d1fd023a65f5d768ee0cb6450706308530f1fca Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sat, 11 Jan 2025 19:18:07 -0700
Subject: [PATCH 3/3] change the place

---
 machines/Depends.nvhpc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
index bb15700a..94d631f6 100644
--- a/machines/Depends.nvhpc
+++ b/machines/Depends.nvhpc
@@ -9,6 +9,10 @@ derivative_mod.o \
 bndry_mod.o \
 prim_advance_mod.o
 
+REDUCED_OPT_OBJS=\
+SatellitePhenologyMod.o \
+esm.o
+
 REDUCED_ERRORCHKS_IN_DEBUG=\
 fvm_consistent_se_cslam.o \
 clm_instMod.o \
@@ -23,10 +27,6 @@ MOM_internal_tide_input.o \
 MOM_ice_shelf.o \
 mom_surface_forcing_nuopc.o
 
-REDUCED_OPT_OBJS=\
-SatellitePhenologyMod.o \
-esm.o
-
 REDUCED_PRECISION_OBJS=\
 shr_wv_sat_mod.o