use AMD-EPYC-based spack-stack

modulefiles/module_gwci.noaacloud.lua

@@ -2,7 +2,7 @@ help([[
 Load environment to run GFS workflow setup scripts on noaacloud
 ]])
 
-prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/contrib/Natalie.Perlin/spack-stack/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core")
 
 load(pathJoin("stack-intel", os.getenv("2021.3.0")))
 load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.3.0")))

modulefiles/module_gwsetup.noaacloud.lua

@@ -4,8 +4,8 @@ Load environment to run GFS workflow setup scripts on noaacloud
 
 load(pathJoin("rocoto"))
 
-prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/apps/modules/modulefiles")
+prepend_path("MODULEPATH", "/contrib/Natalie.Perlin/spack-stack/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core")
+
 load("gnu")
 load("stack-intel")
 load("stack-intel-oneapi-mpi")

parm/config/gefs/config.resources

@@ -41,7 +41,7 @@ case ${machine} in
     ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
-    max_tasks_per_node=36
+    max_tasks_per_node=48
     ;;
   "AZUREPW")
     export PARTITION_BATCH="compute"

scripts/exglobal_cleanup.sh

@@ -8,14 +8,14 @@ echo "Begin Cleanup ${DATAROOT}!"
 # Remove DATAoutput from the forecast model run
 # TODO: Handle this better
 DATAfcst="${DATAROOT}/${RUN}fcst.${PDY:-}${cyc}"
-if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
+#if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
-rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
+#rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 ###############################################################
 
-if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
+#if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
     exit 0
-fi
+#fi
 
 ###############################################################
 # Clean up previous cycles; various depths

scripts/exglobal_forecast.sh

@@ -149,6 +149,11 @@ else
   export OMP_NUM_THREADS=${UFS_THREADS:-1}
 fi
 
+#module load slurm
+
+#export I_MPI_FABRICS=shm:ofi
+#export I_MPI_OFI_PROVIDER=tcp
+
 ${NCP} "${EXECgfs}/${FCSTEXEC}" "${DATA}/"
 ${APRUN_UFS} "${DATA}/${FCSTEXEC}" 1>&1 2>&2
 export ERR=$?

sfsc96.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+ set -x
+
+ source ~/.bashrc
+ source workflow/gw_setup.sh
+
+ HPC_ACCOUNT=${USER} \
+	pslot=c96sfs \
+	RUNTESTS=/lustre/$USER/run \
+	./workflow/create_experiment.py \
+        --yaml sfsc96/sfsc96.yaml
+

sfsc96/options2sfsc96.yaml

@@ -0,0 +1,39 @@
+base:
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "NO"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  WRITE_DOPOST: "YES"
+  KEEPDATA: "YES"
+  DO_EXTRACTVARS: "NO"
+  FHMAX_GFS: 744
+  FHMAX_HF_GFS: 744
+  FHOUT_HF_GFS: 24
+  FHOUT_GFS: 24
+  FHOUT_OCN_GFS: 24
+  FHOUT_ICE_GFS: 24
+  FCST_BREAKPOINTS: ""
+  REPLAY_ICS: "NO"
+  RERUN: "NO"
+  HPSSARCH: "NO"
+  LOCALARCH: "YES"
+  ATSRDIR: "/bucket/Wei.Huang/SFS-REPLAY/archive/sfsc96/1994050100"
+  FLTFILEGFS: "postxconfig-NT-SFS.txt"
+  FLTFILEGFSF00: "postxconfig-NT-SFS.txt"
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  BASE_IC: /bucket/Wei.Huang/SFS-REPLAY_ICs/C96mx100
+fcst:
+  TYPE: "hydro"
+  MONO: "mono"
+  reforecast: "YES"
+  FHZER: 12
+  SFS_POST: "YES"
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "NO"
+  USE_ATM_ENS_PERTURB_FILES: "NO"
+ocn:
+  MOM6_INTERP_ICS: "YES"

sfsc96/sfsc96.yaml

@@ -0,0 +1,21 @@
+experiment:
+  system: gefs
+  mode: forecast-only
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 96
+  resensatmos: 96
+  resdetocean: 1.0
+  nens: 10
+  start: cold
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 1994050100
+  edate: 1994050100
+  yaml: {{ HOMEgfs }}/sfsc96/options2sfsc96.yaml
+
+skip_ci_on_hosts:
+  - wcoss2
+

ush/forecast_predet.sh

@@ -559,6 +559,9 @@ FV3_predet(){
       ${NCP} "${PARMgfs}/post/sfs/postxconfig-NT-sfs.txt"       "${DATA}/postxconfig-NT_FH00.txt"
     fi
   fi
+
+      ${NCP} "${PARMgfs}/post/sfs/postxconfig-NT-sfs.txt"       "${DATA}/postxconfig-NT.txt"
+      ${NCP} "${PARMgfs}/post/sfs/postxconfig-NT-sfs.txt"       "${DATA}/postxconfig-NT_FH00.txt"
 }
 
 # Disable variable not used warnings

versions/build.noaacloud.ver

@@ -1,5 +1,5 @@
-export stack_intel_ver=2021.3.0
-export stack_impi_ver=2021.3.0
+export stack_intel_ver=2021.10.0
+export stack_impi_ver=2021.10.0
 export spack_env=gsi-addon-env
 source "${HOMEgfs:-}/versions/spack.ver"
-export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core"
+export spack_mod_path="/contrib/Natalie.Perlin/spack-stack/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core"

versions/ic.ver

@@ -11,6 +11,7 @@ ic_versions['C48C48mx500']=20241120
 ic_versions['C96']=20241120
 ic_versions['C96C48']=20241120
 ic_versions['C96mx100']=20240610
+ic_versions['C96C96mx100']=20240610
 
 ic_versions['C384C192']=20240610
 ic_versions['C384mx025']=20241120

versions/run.noaacloud.ver

@@ -4,5 +4,6 @@ export spack_env=gsi-addon-env
 
 source "${HOMEgfs:-}/versions/spack.ver"
 export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core"
+export spack_mod_path="/contrib/Natalie.Perlin/spack-stack/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core"
 
 export cdo_ver=2.2.0

workflow/hosts/awspw.yaml

@@ -19,9 +19,9 @@ CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
 BASE_DATA: '/bucket/global-workflow-shared-data'
-BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR'
-LOCALARCH: 'NO'
-ATARDIR: '' # TODO: This will not yet work from AWS.
+BASE_IC: '/bucket/Wei.Huang/SFS-REPLAY_ICs/C96mx100'
+LOCALARCH: 'YES'
+ATARDIR: '/bucket/Wei.Huang/SFS-REPLAY/archive/sfsc96'
 MAKE_NSSTBUFR: 'NO'
 MAKE_ACFTBUFR: 'NO'
 DO_TRACKER: 'NO'