Adding a CI for using Replay ICs with an offset hour of 3

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -15,7 +15,7 @@ arguments:
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312
-  yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_defaults_ci.yaml
 
 skip_ci_on_hosts:
   - wcoss2

ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml

@@ -0,0 +1,19 @@
+experiment:
+  system: gefs
+  mode: forecast-only
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2SWA
+  resdetatmos: 96
+  resdetocean: 1.0
+  resensatmos: 96
+  nens: 2
+  gfs_cyc: 1
+  start: warm
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 2020110100
+  edate: 2020110100
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610

ci/cases/yamls/gefs_replay_ci.yaml

@@ -0,0 +1,14 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
+base:
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  REPLAY_ICS: "YES"
+  FCST_BREAKPOINTS: ""
+  FHMAX_GFS: 48
+  FHMAX_HF_GFS: 24
+  DO_EXTRACTVARS: "YES"
+  FHOUT_HF_GFS: 3
+  FHOUT_OCN_GFS: 24
+  FHOUT_ICE_GFS: 24
+  HOMEDIR: {{ 'RUNTESTS' | getenv }}/GLOBAL
+

jobs/JGLOBAL_WAVE_POST_PNT

@@ -25,7 +25,7 @@ export WAV_MOD_TAG=${RUN}wave${waveMEMB}
 
 export CFP_VERBOSE=1
 
-export FHMAX_WAV_PNT=${FHMAX_WAV}
+export FHMAX_WAV_PNT=$(( FHMAX_WAV - OFFSET_START_HOUR ))
 export DOSPC_WAV='YES' # Spectral post
 export DOBLL_WAV='YES' # Bulletin post
 export DOBNDPNT_WAV='NO'  #not boundary points

parm/archive/gefs_arcdir.yaml.j2

@@ -17,11 +17,17 @@
 
 {% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %}
 
+{% if REPLAY_ICS %}
+    {% set ofst_hr = FHOUT_GFS %}
+{% else %}
+    {% set ofst_hr = FHMIN_GFS %}
+{% endif %}
+
 # Select ensstat files to copy to the arcdir
 {% if RUN == "gefs" %}
     {% set ensstat_files = [] %}
     {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %}
-        {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %}
+        {% for fhr in range(ofst_hr, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %}
             {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~
                                         "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2",
                                         GEFS_ARCH]) %}

parm/config/gefs/config.atmos_products

@@ -16,7 +16,12 @@ export INTERP_ATMOS_MASTERSH="${USHgfs}/interp_atmos_master.sh"
 export INTERP_ATMOS_SFLUXSH="${USHgfs}/interp_atmos_sflux.sh"
 
 export downset=2
-export FHOUT_PGBS=${FHOUT_GFS:-3}  # Output frequency of supplemental gfs pgb file at 1.0 and 0.5 deg
+
+if [[ "${FHMAX_HF_GFS:-0}" == "0" ]]; then
+    export FHOUT_PGBS=${FHOUT_GFS:-3}  # Output frequency of supplemental gfs pgb file at 1.0 and 0.5 deg
+else
+    export FHOUT_PGBS=${FHOUT_HF_GFS:-1}
+fi
 export FLXGF="NO"  # Create interpolated sflux.1p00 file
 
 # paramlist files for the different forecast hours and downsets

parm/config/gefs/config.base

@@ -229,6 +229,15 @@ export FHOUT_ICE=3
 # GFS cycle info
 export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles.
 
+# set variables needed for use with REPLAY ICs
+export REPLAY_ICS=@REPLAY_ICS@
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+    export OFFSET_START_HOUR=$(( assim_freq / 2 ))
+else
+    export OFFSET_START_HOUR=0
+fi
+
+
 # GFS output and frequency
 export FHMIN_GFS=0
 export FHMAX_GFS="@FHMAX_GFS@"
@@ -238,10 +247,10 @@ export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}"
 
 export FHOUT_GFS=6
 export FHMAX_HF_GFS=@FHMAX_HF_GFS@
-export FHOUT_HF_GFS=1
-export FHOUT_OCN_GFS=6
-export FHOUT_ICE_GFS=6
-export FHMIN_WAV=0
+export FHOUT_HF_GFS=@FHOUT_HF_GFS@
+export FHOUT_OCN_GFS=@FHOUT_OCN_GFS@
+export FHOUT_ICE_GFS=@FHOUT_ICE_GFS@
+export FHMIN_WAV=${OFFSET_START_HOUR:-0}
 export FHOUT_WAV=3
 export FHMAX_HF_WAV=120
 export FHOUT_HF_WAV=1
@@ -290,20 +299,9 @@ export NMEM_ENS=@NMEM_ENS@
 export ENSMEM=${ENSMEM:-"000"}
 export MEMDIR="mem${ENSMEM}"
 
-# initialize ocean ensemble members with perturbations
-#   if true, only occurs for members greater than zero
-export REPLAY_ICS=@REPLAY_ICS@
-if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
-    export OFFSET_START_HOUR=$(( assim_freq / 2 ))
-else
-    export OFFSET_START_HOUR=0
-fi
-
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
-# Check if cycle is cold starting
-if [[ "${EXP_WARM_START}" = ".false." ]]; then
-  export IAU_FHROT=${OFFSET_START_HOUR}
-else
+# Check if cycle is warm starting with IAU
+if [[ "${EXP_WARM_START}" = ".true." ]]; then
   if [[ "${DOIAU}" = "YES" ]]; then
     export IAU_FHROT=3
   else

parm/config/gefs/config.efcs

@@ -46,10 +46,6 @@ export SKEB_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"
 export SKEBNORM=1
 export SKEB_NPASS=30
 export SKEB_VDOF=5
-export DO_SHUM="YES"
-export SHUM=0.005
-export SHUM_TAU=21600.
-export SHUM_LSCALE=500000.
 export DO_SPPT="YES"
 export SPPT="0.56,0.28,0.14,0.056,0.028"
 export SPPT_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"

parm/config/gefs/config.extractvars

@@ -14,7 +14,7 @@ export compress_ice=1 #1: Compress extracted ice product, 0: Do not compress ext
 
 export ocnres="1p00" # Resolution of ocean products
 export iceres="native" # Resolution of ice products
-export wavres="0p25" # Resolution of wave products
+export wavres="${waveGRD:4:1}p${waveGRD:5:2}"  # Resolution of wave products
 
 export depthvar_name="z_l" # Name of depth variable in NetCDF ocean products
 export zmin="0." # Minimum depth to extract from NetCDF ocean products

parm/config/gefs/yaml/defaults.yaml

@@ -11,7 +11,11 @@ base:
   DO_EXTRACTVARS: "NO"
   FHMAX_GFS: 120
   FHMAX_HF_GFS: 0
+  FHOUT_HF_GFS: 1
   FCST_BREAKPOINTS: "48"
   REPLAY_ICS: "NO"
   USE_OCN_PERTURB_FILES: "false"
+  FHOUT_OCN_GFS: 6
+  FHOUT_ICE_GFS: 6
+  HPSSARCH: "NO"
   LOCALARCH: "NO"

parm/product/gefs.0p25.fFFF.paramlist.a.txt

@@ -12,13 +12,14 @@
 :CSNOW:surface:
 :CFRZR:surface:
 :CICEP:surface:
+:FDNSSTMP:surface:
 :PWAT:entire atmosphere (considered as a single layer):
 :CAPE:180-0 mb above ground:
 :CAPE:surface:
 :CIN:180-0 mb above ground:
 :CIN:surface:
 :HLCY:3000-0 m above ground:
-:TCDC:entire atmosphere:
+:TCDC:entire atmosphere (considered as a single layer):
 :WEASD:surface:
 :SNOD:surface:
 :ULWRF:top of atmosphere:

parm/product/gefs.0p25.fFFF.paramlist.b.txt

@@ -346,6 +346,7 @@
 :TMP:surface:
 :TMP:tropopause:
 :TOZNE:entire atmosphere (considered as a single layer):
+:TSNOWP:surface:
 :TSOIL:0.1-0.4 m below ground:
 :TSOIL:0.4-1 m below ground:
 :TSOIL:1-2 m below ground:

parm/stage/atmosphere_perturbation.yaml.j2 → parm/stage/atmosphere_ens_perturbations.yaml.j2

@@ -1,12 +1,12 @@
-atmosphere_perturbation:
+atmosphere_ens_perturbation:
     mkdir:
-        {% for mem in range(first_mem, last_mem + 1) %}
+        {% for mem in range(first_mem + 1, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %}
         - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"
         {% endfor %} # mem loop
     copy:
-        {% for mem in range(first_mem, last_mem + 1) %}
+        {% for mem in range(first_mem + 1, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %}
         - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.atminc.nc"]

parm/stage/master_gefs.yaml.j2

@@ -116,7 +116,7 @@
 
 {% if REPLAY_ICS %}
 {% filter indent(width=4) %}
-{% include "atmosphere_perturbation.yaml.j2" %}
+{% include "atmosphere_ens_perturbations.yaml.j2" %}
 {% endfilter %}
 {% endif %}
 
@@ -137,7 +137,7 @@
 {% endif %}
 {% if REPLAY_ICS %}
 {% filter indent(width=4) %}
-{% include "ocean_replay.yaml.j2" %}
+{% include "ocean_ens_perturbations.yaml.j2" %}
 {% endfilter %}
 {% endif %}
 {% if EXP_WARM_START %}

parm/stage/ocean_replay.yaml.j2 → parm/stage/ocean_ens_perturbations.yaml.j2

@@ -1,12 +1,12 @@
-ocean_replay:
+ocean_ens_perturbation:
     mkdir:
-        {% for mem in range(first_mem, last_mem + 1) %}
+        {% for mem in range(first_mem + 1, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %}
         - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}"
         {% endfor %} # mem loop
     copy:
-        {% for mem in range(first_mem, last_mem + 1) %}
+        {% for mem in range(first_mem + 1, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %}
         - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"]

scripts/exgfs_wave_post_pnt.sh

@@ -252,7 +252,9 @@ source "${USHgfs}/preamble.sh"
                                ww3_outp_spec.inp.tmpl > ww3_outp.inp
 
     ${NLN} mod_def.$waveuoutpGRD mod_def.ww3
-    HMS="${cyc}0000"
+    #export OFFSET_START_HOUR=$( printf "%02d" ${half_assim} )
+    hh=$( printf "%02d" $(( cyc + OFFSET_START_HOUR )) )
+    HMS="${hh}0000"
     if [[ -f "${COMIN_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" ]]; then
       ${NLN} "${COMIN_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" \
         "./out_pnt.${waveuoutpGRD}"

ush/extractvars_tools.sh

@@ -17,7 +17,8 @@ check_atmos() {
   done
   mapfile -t requestedvar_in_allgrb2file_arr < "${requestedvar_in_allgrb2file}"
   while read -r vari; do
-    if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ ${vari} ]] ;then
+    # shellcheck disable=SC2076
+    if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ "${vari}" ]] ;then
       echo "WARNING: PARM VARIABLE (${vari}) is not available in pgrb and pgrb2b for f${fnhl}."
     fi
   done <"${varlistl}"

ush/ocnice_extractvars.sh

@@ -21,15 +21,23 @@ comout_rfcst_prod_ocnice=${6}
 
 [[ -d "${subdata}" ]] || mkdir -p "${subdata}"
 
-for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do
+for (( nh = FHMIN_GFS + fhout_ocnice; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do
   fnh=$(printf "%3.3d" "${nh}")
 
   if [[ ${component_name} == "ocn" ]]; then
-    infile=${COMIN_OCEAN_NETCDF}/${datares}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc
+    if [[ "${datares}" == "native" ]]; then
+      infile="${COMIN_OCEAN_HISTORY}/${RUN}.ocean.t${cyc}z.${fhout_ocnice}hr_avg.f${fnh}.nc"
+    else
+      infile="${COMIN_OCEAN_NETCDF}/${datares}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc"
+    fi
     # For ocean products, add an argument to extract a subset of levels
     otherargs=(-d "${depthvar_name},""${zmin},""${zmax}")
   elif [[ ${component_name} == "ice" ]]; then
-    infile=${COMIN_ICE_NETCDF}/${datares}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc
+    if [[ "${datares}" == "native" ]]; then
+      infile="${COMIN_ICE_HISTORY}/${RUN}.ice.t${cyc}z.${fhout_ocnice}hr_avg.f${fnh}.nc"
+    else
+      infile="${COMIN_ICE_NETCDF}/${datares}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc"
+    fi
     otherargs=()
   fi
   outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc

ush/parsing_namelists_MOM6.sh

@@ -41,6 +41,12 @@ cat input.nml
 # ---------
 # Prepare local variables for use in MOM_input.IN from UFSWM
 # The ones already defined are left commented as a reminder
+# == MOM options to start from coarsed grained restarts, set to off by default
+#       options only available for 05 and 1 degree grids
+#       as restarts are coarsed grained/interpolated from the 0.25 degrees grid
+local MOM6_INIT_FROM_Z=${MOM6_INIT_FROM_Z:-True}
+local MOM6_WARMSTART_FILE=${MOM6_WARMSTART_FILE:-"none"}
+local MOM6_INIT_UV=${MOM6_INIT_UV:-"zero"}
 # == MOM_domains section ==
 # NX_GLB
 # NY_GLB

workflow/rocoto/gefs_tasks.py

@@ -314,6 +314,7 @@ def atmos_ensstat(self):
 
     def wavepostsbs(self):
         deps = []
+
         dep_dict = {'type': 'metatask', 'name': f'fcst_mem#member#'}
         deps.append(rocoto.add_dependency(dep_dict))
         dependencies = rocoto.create_dependency(dep=deps)