make AWS use different partitions for fcst and others

NOAA-EPIC · Nov 21, 2024 · b078484 · b078484
1 parent 0502044
commit b078484
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 6 deletions.
diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
@@ -41,7 +41,7 @@ case ${machine} in
     ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
-    max_tasks_per_node=36
+    max_tasks_per_node=48
     ;;
   "AZUREPW")
     export PARTITION_BATCH="compute"

diff --git a/parm/config/gefs/config.resources.AWSPW b/parm/config/gefs/config.resources.AWSPW
@@ -9,3 +9,63 @@ unset memory
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
   unset "${mem_var}"
 done
+
+step=$1
+
+echo "step: $step"
+
+case ${step} in
+  "fcst" | "efcs")
+    export PARTITION_BATCH="compute"
+    max_tasks_per_node=48
+    ;;
+
+  "arch")
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    ;;
+
+  "prep_emissions")
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    export ntasks=1
+    export threads_per_task=1
+    export tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    ;;
+
+  "waveinit")
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    export ntasks=12
+    export threads_per_task=1
+    export tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    export NTASKS=${ntasks}
+    ;;
+
+  "wavepostpnt")
+    export PARTITION_BATCH="compute"
+    max_tasks_per_node=48
+    export ntasks=240
+    export threads_per_task=1
+    export tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    export NTASKS=${ntasks}
+    ;;
+
+  "wavepostsbs" | "wavepostbndpnt" | "wavepostbndpntbll")
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    export ntasks=24
+    export threads_per_task=1
+    export tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    export NTASKS=${ntasks}
+    ;;
+
+  *)
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    ;;
+
+esac
+
+export max_tasks_per_node
+
diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
@@ -483,4 +483,20 @@ export OFFSET_START_HOUR=0
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 
+# The tracker, genesis, and METplus jobs are not supported on CSPs yet
+# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS/AZURE setup, not for general.
+if [[ "${machine}" =~ "PW" ]]; then
+   export DO_TRACKER="NO"
+   export DO_GENESIS="NO"
+   export DO_METP="NO"
+   export DO_WAVE="NO"
+fi
+
+# The tracker and genesis are not installed on Orion/Hercules yet; this requires spack-stack builds of the package.
+# TODO: we should place these in workflow/hosts/[orion|hercules].yaml.
+if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then
+   export DO_TRACKER="NO"
+   export DO_GENESIS="NO"
+fi
+
 echo "END: config.base"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -107,8 +107,8 @@ case ${machine} in
     ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
-    npe_node_max=36
-    max_tasks_per_node=36
+    npe_node_max=48
+    max_tasks_per_node=48
     # TODO Supply a max mem/node value for AWS
     # shellcheck disable=SC2034
     mem_node_max=""

diff --git a/parm/config/gfs/config.resources.AWSPW b/parm/config/gfs/config.resources.AWSPW
@@ -9,3 +9,27 @@ unset memory
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
   unset "${mem_var}"
 done
+
+step=$1
+
+case ${step} in
+  "fcst" | "efcs")
+    export PARTITION_BATCH="compute"
+    max_tasks_per_node=48
+    ;;
+
+  "arch")
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    ;;
+
+
+  *)
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    ;;
+
+esac
+
+export max_tasks_per_node
+
diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd
diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd
diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
+8 −5		modulefiles/gfsutils_noaacloud.intel.lua
+1 −1		src/ocnicepost.fd/masking_mod.F90
+5 −0		ush/module-setup.sh
+4 −4		docs/source/chgres_cube.rst
+9 −9		docs/source/ufs_utils.rst
+5 −9		fix/link_fixdirs.sh
+64 −0		modulefiles/build.gaeaC6.intel.lua
+7 −10		modulefiles/build.noaacloud.intel.lua
+0 −10		modulefiles/common4noaacloud.lua
+1 −1		reg_tests/cpld_gridgen/rt.sh
+1 −3		reg_tests/ice_blend/driver.jet.sh
+4 −1		sorc/machine-setup.sh
+3 −11		sorc/orog_mask_tools.fd/orog.fd/CMakeLists.txt
+319 −20		sorc/orog_mask_tools.fd/orog.fd/io_utils.F90
+0 −4,468		sorc/orog_mask_tools.fd/orog.fd/mtnlm7_oclsm.F
+1,365 −0		sorc/orog_mask_tools.fd/orog.fd/mtnlm7_oclsm.F90
+1,104 −0		sorc/orog_mask_tools.fd/orog.fd/orog_utils.F90
+1 −0		tests/CMakeLists.txt
+23 −0		tests/orog/CMakeLists.txt
+58 −0		tests/orog/ftst_get_ll_angle.F90
+87 −0		tests/orog/ftst_ll2xyz.F90
+44 −0		tests/orog/ftst_minmax.F90
+1 −1		ush/fv3gfs_driver_grid.sh
+1 −1		util/gdas_init/set_fixed_files.sh