From c0b6d13a53b2da3a5cf7e61c1ddf47015b9d7194 Mon Sep 17 00:00:00 2001
From: DavidBurrows-NCO <82525974+DavidBurrows-NCO@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:31:41 -0500
Subject: [PATCH] Add/fix build capability for Gaea-C5, Gaea-C6, and container
 (#87)

After the recent Gaea-C5 OS upgrade, gfs_utils fails to build.
This corrects Gaea-C5 build, adds Gaea-C6 build capability (following
ufs-wx-model
[2448](https://github.com/ufs-community/ufs-weather-model/pull/2448)),
and adds containerized build capability.

Refs NOAA-EMC/global-workflow
[3011](https://github.com/NOAA-EMC/global-workflow/issues/3011)
Refs NOAA-EMC/global-workflow
[3025](https://github.com/NOAA-EMC/global-workflow/issues/3025)
Resolve #86
---------

Co-authored-by: Mark A Potts <mark.potts@noaa.gov>
---
 CMakeLists.txt                                |  1 +
 modulefiles/gfsutils_container.intel.lua      | 17 ++++++++++++
 ...ea.intel.lua => gfsutils_gaeac5.intel.lua} | 15 ++++-------
 modulefiles/gfsutils_gaeac6.intel.lua         | 27 +++++++++++++++++++
 src/tave.fd/CMakeLists.txt                    |  7 ++++-
 src/vint.fd/CMakeLists.txt                    | 11 +++++---
 ush/detect_machine.sh                         | 21 ++++++++++-----
 ush/module-setup.sh                           |  7 +++++
 8 files changed, 86 insertions(+), 20 deletions(-)
 create mode 100644 modulefiles/gfsutils_container.intel.lua
 rename modulefiles/{gfsutils_gaea.intel.lua => gfsutils_gaeac5.intel.lua} (54%)
 create mode 100644 modulefiles/gfsutils_gaeac6.intel.lua

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7a213736..2493be0a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,6 +25,7 @@ endif()
 
 # User options.
 option(OPENMP "use OpenMP threading" ON)
+option(LINK_W3EMC "Link w3emc library" ON)
 
 # Find packages.
 find_package(MPI REQUIRED)
diff --git a/modulefiles/gfsutils_container.intel.lua b/modulefiles/gfsutils_container.intel.lua
new file mode 100644
index 00000000..cddef43a
--- /dev/null
+++ b/modulefiles/gfsutils_container.intel.lua
@@ -0,0 +1,17 @@
+help([[
+Build environment for GFS utilities in a container 
+]])
+
+prepend_path("MODULEPATH", "/opt/spack-stack/spack-stack-1.8.0/envs/unified-env/install/modulefiles/Core")
+
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0"
+local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.12.1"
+local cmake_ver=os.getenv("cmake_ver") or "3.27.9"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+load(pathJoin("cmake", cmake_ver))
+
+load("gfsutils_common")
+
+whatis("Description: GFS utilities environment in container with Intel Compilers")
diff --git a/modulefiles/gfsutils_gaea.intel.lua b/modulefiles/gfsutils_gaeac5.intel.lua
similarity index 54%
rename from modulefiles/gfsutils_gaea.intel.lua
rename to modulefiles/gfsutils_gaeac5.intel.lua
index 0d7f9df0..7844b57e 100644
--- a/modulefiles/gfsutils_gaea.intel.lua
+++ b/modulefiles/gfsutils_gaeac5.intel.lua
@@ -1,32 +1,27 @@
 help([[
   This module loads libraries required for building and running GFS UTILS 
-  on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+  on the NOAA RDHPC machine Gaea C5 using Intel-2023.2.0.
 ]])
 
-whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
+whatis([===[Loads libraries needed for building the GFS utilities on Gaea C5]===])
 
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
 
-stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
 load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.11.6"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
 load("gfsutils_common")
 load("nccmp/1.9.0.1")
 
-unload("darshan-runtime")
 unload("cray-libsci")
 
 setenv("CC","cc")
 setenv("CXX","CC")
 setenv("FC","ftn")
-setenv("CMAKE_Platform","gaea.intel")
+setenv("CMAKE_Platform","gaeac5.intel")
diff --git a/modulefiles/gfsutils_gaeac6.intel.lua b/modulefiles/gfsutils_gaeac6.intel.lua
new file mode 100644
index 00000000..619caa99
--- /dev/null
+++ b/modulefiles/gfsutils_gaeac6.intel.lua
@@ -0,0 +1,27 @@
+help([[
+  This module loads libraries required for building and running GFS UTILS 
+  on the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0.
+]])
+
+whatis([===[Loads libraries needed for building the GFS utilities on Gaea C6 ]===])
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("gfsutils_common")
+load("nccmp/1.9.0.1")
+
+unload("cray-libsci")
+
+setenv("CC","cc")
+setenv("CXX","CC")
+setenv("FC","ftn")
+setenv("CMAKE_Platform","gaeac6.intel")
diff --git a/src/tave.fd/CMakeLists.txt b/src/tave.fd/CMakeLists.txt
index 70905a0b..1d4cd1d4 100644
--- a/src/tave.fd/CMakeLists.txt
+++ b/src/tave.fd/CMakeLists.txt
@@ -10,8 +10,13 @@ endif()
 
 set(exe_name tave.x)
 add_executable(${exe_name} ${fortran_src})
-target_link_libraries(${exe_name} PRIVATE bacio::bacio_4
+if(LINK_W3EMC)
+  target_link_libraries(${exe_name} PRIVATE bacio::bacio_4
                                           w3emc::w3emc_d
                                           g2::g2_d)
+else()
+  target_link_libraries(${exe_name} PRIVATE bacio::bacio_4
+                                          g2::g2_d)
+endif()
 
 install(TARGETS ${exe_name} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/src/vint.fd/CMakeLists.txt b/src/vint.fd/CMakeLists.txt
index 118faab2..a9d21699 100644
--- a/src/vint.fd/CMakeLists.txt
+++ b/src/vint.fd/CMakeLists.txt
@@ -10,8 +10,13 @@ endif()
 
 set(exe_name vint.x)
 add_executable(${exe_name} ${fortran_src})
-target_link_libraries(${exe_name} PRIVATE bacio::bacio_4
-                                          w3emc::w3emc_d
-                                          g2::g2_d)
+if(LINK_W3EMC)
+  target_link_libraries(${exe_name} PRIVATE bacio::bacio_4
+                                            w3emc::w3emc_d
+                                            g2::g2_d)
+else()
+  target_link_libraries(${exe_name} PRIVATE bacio::bacio_4
+                                            g2::g2_d)
+endif()
 
 install(TARGETS ${exe_name} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh
index b049a604..4167d617 100755
--- a/ush/detect_machine.sh
+++ b/ush/detect_machine.sh
@@ -21,8 +21,11 @@ case $(hostname -f) in
   dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9
   dlogin10.dogwood.wcoss2.ncep.noaa.gov)     MACHINE_ID=wcoss2 ;; ### dogwood10
 
-  gaea5[1-8])          MACHINE_ID=gaea ;; ### gaea51-58
-  gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58
+  gaea5[1-8])          MACHINE_ID=gaeac5 ;; ### gaea51-58
+  gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58
+
+  gaea6[1-8])          MACHINE_ID=gaeac6 ;; ### gaea61-68
+  gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68
 
   hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09
   hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12
@@ -61,7 +64,10 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then
 fi
 
 # Try searching based on paths since hostname may not match on compute nodes
-if [[ -d /lfs/h3 ]]; then
+if [[ -d /opt/spack-stack ]]; then
+  # We are in a container
+  MACHINE_ID=container
+elif [[ -d /lfs/h3 ]]; then
   # We are on NOAA Cactus or Dogwood
   MACHINE_ID=wcoss2
 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then
@@ -81,9 +87,12 @@ elif [[ -d /work ]]; then
   else
     MACHINE_ID=orion
   fi
-elif [[ -d /gpfs && -d /ncrc ]]; then
-  # We are on GAEA.
-  MACHINE_ID=gaea
+elif [[ -d /gpfs/f5 ]]; then
+  # We are on GAEAC5.
+  MACHINE_ID=gaeac5
+elif [[ -d /gpfs/f6 ]]; then
+  # We are on GAEAC6.
+  MACHINE_ID=gaeac6
 elif [[ -d /data/prod ]]; then
   # We are on SSEC's S4
   MACHINE_ID=s4
diff --git a/ush/module-setup.sh b/ush/module-setup.sh
index a4635808..14904aee 100755
--- a/ush/module-setup.sh
+++ b/ush/module-setup.sh
@@ -52,6 +52,13 @@ elif [[ $MACHINE_ID = cheyenne* ]] ; then
     fi
     module purge
 
+elif [[ $MACHINE_ID = container ]] ; then
+    # We are in a container
+    if ( ! eval module help > /dev/null 2>&1 ) ; then
+        source /usr/lmod/lmod/init/bash
+    fi
+    module purge
+
 elif [[ $MACHINE_ID = stampede* ]] ; then
     # We are on TACC Stampede
     if ( ! eval module help > /dev/null 2>&1 ) ; then