From 3d5da07cefeaf6374b7c4f13daf88e0d5a1901e7 Mon Sep 17 00:00:00 2001
From: RussTreadon-NOAA <russ.treadon@noaa.gov>
Date: Wed, 26 Jun 2024 15:20:36 +0000
Subject: [PATCH 1/2] initial changes to build and run GSI on Orion Rocky 9
 (#754)

---
 modulefiles/gsi_orion.intel.lua | 9 +++++----
 regression/regression_var.sh    | 9 +++++++--
 ush/detect_machine.sh           | 4 ++--
 ush/sub_orion                   | 8 +++++---
 4 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/modulefiles/gsi_orion.intel.lua b/modulefiles/gsi_orion.intel.lua
index 03ea22018d..d05bda5b2e 100644
--- a/modulefiles/gsi_orion.intel.lua
+++ b/modulefiles/gsi_orion.intel.lua
@@ -1,11 +1,11 @@
 help([[
 ]])
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env-rocky9/install/modulefiles/Core")
 
-local stack_python_ver=os.getenv("python_ver") or "3.11.6"
-local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
-local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
+local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6"
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
+local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
 local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
 
@@ -16,6 +16,7 @@ load(pathJoin("cmake", cmake_ver))
 
 load("gsi_common")
 load(pathJoin("prod_util", prod_util_ver))
+load("intel-oneapi-mkl/2022.2.1")
 
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")
diff --git a/regression/regression_var.sh b/regression/regression_var.sh
index aebbccab8b..cc4cdeaab3 100755
--- a/regression/regression_var.sh
+++ b/regression/regression_var.sh
@@ -40,8 +40,13 @@ elif [[ -d /ncrc ]]; then # Gaea
   export machine="Gaea"
 elif [[ -d /data/prod ]]; then # S4
   export machine="S4"
-elif [[ -d /work && $(hostname) =~ "Orion" ]]; then # Orion
-  export machine="Orion"
+elif [[ -d /work ]]; then # Orion or Hercules
+  mount=$(findmnt -n -o SOURCE /home)
+  if [[ ${mount} =~ "hercules" ]]; then
+    export machine="Hercules"
+  else
+    export machine="Orion"
+  fi
 elif [[ -d /work && $(hostname) =~ "hercules" ]]; then # Hercules
   export machine="Hercules"
 elif [[ -d /lfs/h2 ]]; then # wcoss2
diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh
index 683ee0db7f..e275dcccee 100755
--- a/ush/detect_machine.sh
+++ b/ush/detect_machine.sh
@@ -75,8 +75,8 @@ elif [[ -d /scratch1 ]]; then
   MACHINE_ID=hera
 elif [[ -d /work ]]; then
   # We are on MSU Orion or Hercules
-  if [[ -d /apps/other ]]; then
-    # We are on Hercules
+  mount=$(findmnt -n -o SOURCE /home)
+  if [[ ${mount} =~ "hercules" ]]; then
     MACHINE_ID=hercules
   else
     MACHINE_ID=orion
diff --git a/ush/sub_orion b/ush/sub_orion
index b810576379..371c30e321 100755
--- a/ush/sub_orion
+++ b/ush/sub_orion
@@ -115,23 +115,25 @@ echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --exclusive"    >> $cfile
 echo "#SBATCH --account=$accnt"                                                    >> $cfile
 
 echo "" >>$cfile
+echo "set -x" >> $cfile
 echo "export ntasks=$(( $nodes * $procs ))" >> $cfile
 echo "export ppn=$procs"                    >> $cfile
 echo "export threads=$threads"              >> $cfile
 echo "export OMP_NUM_THREADS=$threads"      >> $cfile
-##echo "export OMP_STACKSIZE=2048M"           >> $cfile
 echo "ulimit -s unlimited"                  >> $cfile
 
 echo "" >>$cfile
 echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out)  >>$cfile
 echo "" >>$cfile
 
-echo ". /apps/lmod/lmod/init/sh"                           >> $cfile
+echo ". /apps/other/lmod/lmod/init/sh"                     >> $cfile
 echo "module purge"                                        >> $cfile
 echo "module use $modulefiles"                      >> $cfile
 echo "module load gsi_orion.intel" >> $cfile
 echo "module list"                                         >> $cfile
-echo ""                                                    >> $cfile
+#TODO reenable I_MPI_EXTRA_FILESYSTEM once regional ctests can properly handle parallel I/O on Orion
+echo "unset I_MPI_EXTRA_FILESYSTEM"         >> $cfile
+
 cat $exec >> $cfile
 
 if [[ $nosub = YES ]];then

From 9c2cbf8146e9fa0bb187c8f2b67bd38a5d77231a Mon Sep 17 00:00:00 2001
From: RussTreadon-NOAA <russ.treadon@noaa.gov>
Date: Fri, 28 Jun 2024 14:05:09 +0000
Subject: [PATCH 2/2] remove unnecessary hercules check in regression_var.sh
 (#754)

---
 regression/regression_var.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/regression/regression_var.sh b/regression/regression_var.sh
index 6a53c6d9da..4a2bc85874 100755
--- a/regression/regression_var.sh
+++ b/regression/regression_var.sh
@@ -47,8 +47,6 @@ elif [[ -d /work ]]; then # Orion or Hercules
   else
     export machine="Orion"
   fi
-elif [[ -d /work && $(hostname) =~ "hercules" ]]; then # Hercules
-  export machine="Hercules"
 elif [[ -d /lfs/h2 ]]; then # wcoss2
   export machine="wcoss2"
 fi