diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index f663d02895..70f8a2ad89 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -6,108 +6,111 @@ # Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer) {% if IAUFHRS is string %} -# "3,6,9" -{% set iaufhrs = [] %} -{% for iaufhr in IAUFHRS.split(",") %} -{% do iaufhrs.append(iaufhr | int) %} -{% endfor %} + # "3,6,9" + {% set iaufhrs = [] %} + {% for iaufhr in IAUFHRS.split(",") %} + {% do iaufhrs.append(iaufhr | int) %} + {% endfor %} {% else %} -# 6 (integer) -{% set iaufhrs = [IAUFHRS] %} + # 6 (integer) + {% set iaufhrs = [IAUFHRS] %} {% endif %} # Repeat for IAUFHRS_ENKF {% if IAUFHRS_ENKF is string %} -{% set iaufhrs_enkf = [] %} -{% for iaufhr in IAUFHRS_ENKF.split(",") %} -{% do iaufhrs_enkf.append(iaufhr | int) %} -{% endfor %} + {% set iaufhrs_enkf = [] %} + {% for iaufhr in IAUFHRS_ENKF.split(",") %} + {% do iaufhrs_enkf.append(iaufhr | int) %} + {% endfor %} {% else %} -{% set iaufhrs_enkf = [IAUFHRS_ENKF] %} + {% set iaufhrs_enkf = [IAUFHRS_ENKF] %} {% endif %} # Determine which data to archive datasets: {% if ENSGRP == 0 %} -{% filter indent(width=4) %} -# Archive the ensemble means and spreads + {% filter indent(width=4) %} + # Archive the ensemble means and spreads {% include "enkf.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% else %} -# Archive individual member data -# First, construct individual member directories from templates -# COMIN_ATMOS_ANALYSIS_MEM, COMIN_ATMOS_HISTORY_MEM, and COMIN_ATMOS_RESTART_MEM - -# Declare to-be-filled lists of member COM directories -{% set COMIN_ATMOS_ANALYSIS_MEM_list = [] %} -{% set COMIN_ATMOS_RESTART_MEM_list = [] %} -{% set COMIN_ATMOS_HISTORY_MEM_list = [] %} - -# Determine which ensemble members belong to this group -{% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} -{% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, nmem_ens ] | min %} - -# Construct member COM directories for the group -{% for mem in range(first_group_mem, last_group_mem + 1) %} - -# Declare a dict of search and replace terms to run on each template -{% set tmpl_dict = {'ROTDIR':ROTDIR, - 'RUN':RUN, - 'YMD':cycle_YMD, - 'HH':cycle_HH, - 'MEMDIR':"mem" + '%03d' % mem} %} - -# Replace template variables with tmpl_dict, one key at a time -# This must be done in a namespace to overcome jinja scoping -# Variables set inside of a for loop are lost at the end of the loop -# unless they are part of a namespace -{% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL, - COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL, - COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %} - -{% for key in tmpl_dict.keys() %} -{% set search_term = '${' + key + '}' %} -{% set replace_term = tmpl_dict[key] %} -{% set com_ns.COMIN_ATMOS_ANALYSIS_MEM = com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %} -{% set com_ns.COMIN_ATMOS_HISTORY_MEM = com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %} -{% set com_ns.COMIN_ATMOS_RESTART_MEM = com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %} -{% endfor %} - -# Append the member COM directories -{% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%} -{% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%} -{% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%} - -{% endfor %} - -# Archive member data -{% filter indent(width=4) %} + # Archive individual member data + # First, construct individual member directories from templates + # COMIN_ATMOS_ANALYSIS_MEM, COMIN_ATMOS_HISTORY_MEM, and COMIN_ATMOS_RESTART_MEM + + # Declare to-be-filled lists of member COM directories + {% set COMIN_ATMOS_ANALYSIS_MEM_list = [] %} + {% set COMIN_ATMOS_RESTART_MEM_list = [] %} + {% set COMIN_ATMOS_HISTORY_MEM_list = [] %} + + # Determine which ensemble members belong to this group + {% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} + {% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, nmem_ens ] | min %} + + # Construct member COM directories for the group + {% for mem in range(first_group_mem, last_group_mem + 1) %} + + # Declare a dict of search and replace terms to run on each template + {% set tmpl_dict = {'ROTDIR':ROTDIR, + 'RUN':RUN, + 'YMD':cycle_YMD, + 'HH':cycle_HH, + 'MEMDIR':"mem" + '%03d' % mem} %} + + # Replace template variables with tmpl_dict, one key at a time + # This must be done in a namespace to overcome jinja scoping + # Variables set inside of a for loop are lost at the end of the loop + # unless they are part of a namespace + {% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL, + COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL, + COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %} + + {% for key in tmpl_dict.keys() %} + {% set search_term = '${' + key + '}' %} + {% set replace_term = tmpl_dict[key] %} + {% set com_ns.COMIN_ATMOS_ANALYSIS_MEM = + com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %} + {% set com_ns.COMIN_ATMOS_HISTORY_MEM = + com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %} + {% set com_ns.COMIN_ATMOS_RESTART_MEM = + com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %} + {% endfor %} + + # Append the member COM directories + {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%} + {% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%} + {% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%} + + {% endfor %} + + # Archive member data + {% filter indent(width=4) %} {% include "enkf_grp.yaml.j2" %} -{% endfilter %} + {% endfilter %} -# Determine if restarts should be saved -{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} + # Determine if restarts should be saved + {% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} -# Save the increments and restarts every ARCH_WARMICFREQ days -# The ensemble increments (group a) should be saved on the ARCH_CYC -{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} -{% if ARCH_CYC == cycle_HH | int %} -{% filter indent(width=4) %} + # Save the increments and restarts every ARCH_WARMICFREQ days + # The ensemble increments (group a) should be saved on the ARCH_CYC + {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} + {% if ARCH_CYC == cycle_HH | int %} + {% filter indent(width=4) %} {% include "enkf_restarta_grp.yaml.j2" %} -{% endfilter %} -{% endif %} -{% endif %} - -# The ensemble ICs (group b) are restarts and always lag increments by assim_freq -{% set ics_offset = (assim_freq | string + "H") | to_timedelta %} -{% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 %} -{% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int %} -{% filter indent(width=4) %} + {% endfilter %} + {% endif %} + {% endif %} + + # The ensemble ICs (group b) are restarts and always lag increments by assim_freq + {% set ics_offset = (assim_freq | string + "H") | to_timedelta %} + {% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 %} + {% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int %} + {% filter indent(width=4) %} {% include "enkf_restartb_grp.yaml.j2" %} -{% endfilter %} -{% endif %} -{% endif %} + {% endfilter %} + {% endif %} + {% endif %} -# End of individual member archiving + # End of individual member archiving {% endif %} diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index f25fd9de40..30a2175653 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -5,12 +5,12 @@ # Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer) {% if IAUFHRS is string %} -{% set iaufhrs = [] %} -{% for iaufhr in IAUFHRS.split(",") %} -{% do iaufhrs.append(iaufhr | int) %} -{% endfor %} + {% set iaufhrs = [] %} + {% for iaufhr in IAUFHRS.split(",") %} + {% do iaufhrs.append(iaufhr | int) %} + {% endfor %} {% else %} -{% set iaufhrs = [IAUFHRS] %} + {% set iaufhrs = [IAUFHRS] %} {% endif %} datasets: @@ -20,84 +20,90 @@ datasets: {% endfilter %} {% if DO_ICE %} -# Ice data -{% filter indent(width=4) %} + # Ice data + {% filter indent(width=4) %} {% include "gdasice.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% endif %} {% if DO_OCN %} -# Ocean forecast products -{% filter indent(width=4) %} + # Ocean forecast products + {% filter indent(width=4) %} {% include "gdasocean.yaml.j2" %} -{% endfilter %} -{% if DO_JEDIOCNVAR and MODE == "cycled" %} -# Ocean analysis products -{% filter indent(width=4) %} + {% endfilter %} + {% if DO_JEDIOCNVAR and MODE == "cycled" %} + # Ocean analysis products + {% filter indent(width=4) %} {% include "gdasocean_analysis.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} {% endif %} {% if DO_WAVE %} -# Wave products -{% filter indent(width=4) %} + # Wave products + {% filter indent(width=4) %} {% include "gdaswave.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% endif %} {% if MODE == "cycled" %} -# Determine if we will save restart ICs or not (only valid for cycled) -{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} + # Determine if we will save restart ICs or not (only valid for cycled) + {% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} -{% if ARCH_CYC == cycle_HH | int%} -# Save the warm and forecast-only cycle ICs every ARCH_WARMICFREQ days -{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} -{% set save_warm_start_forecast = True %} -{% set save_warm_start_cycled = True %} -# Save the forecast-only restarts every ARCH_FCSTICFREQ days -{% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} -{% set save_warm_start_forecast = True %} -{% endif %} -{% endif %} + {% if ARCH_CYC == cycle_HH | int%} + # Save the forecast-only cycle ICs every ARCH_WARMICFREQ or ARCH_FCSTICFREQ days + {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} + {% set save_warm_start_forecast = True %} + {% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} + {% set save_warm_start_forecast = True %} + {% endif %} + {% endif %} -{% if save_warm_start_forecast %} -# Save warm start forecast-only data -# Atmosphere restarts -{% filter indent(width=4) %} + # The GDAS ICs (group b) are restarts and always lag increments by assim_freq + {% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int %} + {% set ics_offset = (assim_freq | string + "H") | to_timedelta %} + {% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 %} + {% set save_warm_start_cycled = True %} + {% endif %} + {% endif %} + + {% if save_warm_start_forecast %} + # Save warm start forecast-only data + # Atmosphere restarts + {% filter indent(width=4) %} {% include "gdas_restarta.yaml.j2" %} -{% endfilter %} + {% endfilter %} -{% if DO_WAVE %} -# Wave restarts -{% filter indent(width=4) %} + {% if DO_WAVE %} + # Wave restarts + {% filter indent(width=4) %} {% include "gdaswave_restart.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} -{% if DO_OCN %} -# Ocean restarts -{% filter indent(width=4) %} + {% if DO_OCN %} + # Ocean restarts + {% filter indent(width=4) %} {% include "gdasocean_restart.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} -{% if DO_ICE %} -# Ice restarts -{% filter indent(width=4) %} + {% if DO_ICE %} + # Ice restarts + {% filter indent(width=4) %} {% include "gdasice_restart.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} -# End of forecast-only restarts -{% endif %} + # End of forecast-only restarts + {% endif %} -{% if save_warm_start_cycled %} -# Save warm start cycled restarts -{% filter indent(width=4) %} + {% if save_warm_start_cycled %} + # Save warm start cycled restarts + {% filter indent(width=4) %} {% include "gdas_restartb.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} -# End of restart checking + # End of restart checking {% endif %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index 67cde482a2..14178f3e7e 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -5,14 +5,14 @@ # Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer) {% if IAUFHRS is string %} -# "3,6,9" -{% set iaufhrs = [] %} -{% for iaufhr in IAUFHRS.split(",") %} -{% do iaufhrs.append(iaufhr | int) %} -{% endfor %} + # "3,6,9" + {% set iaufhrs = [] %} + {% for iaufhr in IAUFHRS.split(",") %} + {% do iaufhrs.append(iaufhr | int) %} + {% endfor %} {% else %} -# 6 (integer) -{% set iaufhrs = [IAUFHRS] %} + # 6 (integer) + {% set iaufhrs = [IAUFHRS] %} {% endif %} # Determine which data to archive @@ -24,89 +24,89 @@ datasets: {% endfilter %} {% if ARCH_GAUSSIAN %} -# Archive Gaussian data -{% filter indent(width=4) %} + # Archive Gaussian data + {% filter indent(width=4) %} {% include "gfs_flux.yaml.j2" %} {% include "gfs_netcdfb.yaml.j2" %} {% include "gfs_pgrb2b.yaml.j2" %} -{% endfilter %} -{% if MODE == "cycled" %} -# Archive Gaussian analysis data -{% filter indent(width=4) %} + {% endfilter %} + {% if MODE == "cycled" %} + # Archive Gaussian analysis data + {% filter indent(width=4) %} {% include "gfs_netcdfa.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} {% endif %} {% if DO_WAVE %} -# Wave forecasts -{% filter indent(width=4) %} + # Wave forecasts + {% filter indent(width=4) %} {% include "gfswave.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% endif %} {% if AERO_FCST_CDUMP == "gfs" or AERO_FCST_CDUMP == "both" %} -# Aerosol forecasts -{% filter indent(width=4) %} + # Aerosol forecasts + {% filter indent(width=4) %} {% include "chem.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% endif %} {% if DO_OCN %} -# Ocean forecasts -{% filter indent(width=4) %} + # Ocean forecasts + {% filter indent(width=4) %} {% include "ocean_6hravg.yaml.j2" %} {% include "ocean_grib2.yaml.j2" %} {% include "gfs_flux_1p00.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% endif %} {% if DO_ICE %} -# Ice forecasts -{% filter indent(width=4) %} + # Ice forecasts + {% filter indent(width=4) %} {% include "ice_6hravg.yaml.j2" %} {% include "ice_grib2.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% endif %} {% if DO_BUFRSND %} -# Downstream BUFR soundings -{% filter indent(width=4) %} + # Downstream BUFR soundings + {% filter indent(width=4) %} {% include "gfs_downstream.yaml.j2" %} -{% endfilter %} + {% endfilter %} {% endif %} # Determine whether to save the MOS tarball {% if DO_MOS and cycle_HH == "18" %} -{% if not REALTIME %} -{% filter indent(width=4) %} + {% if not REALTIME %} + {% filter indent(width=4) %} {% include "gfsmos.yaml.j2" %} -{% endfilter %} + {% endfilter %} -{% else %} + {% else %} -{% set td_from_sdate = current_cycle - SDATE %} -{% set td_one_day = "+1D" | to_timedelta %} -{% if td_from_sdate > td_one_day %} -{% filter indent(width=4) %} + {% set td_from_sdate = current_cycle - SDATE %} + {% set td_one_day = "+1D" | to_timedelta %} + {% if td_from_sdate > td_one_day %} + {% filter indent(width=4) %} {% include "gfsmos.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} -{% endif %} + {% endif %} {% endif %} # Determine if we will save restart ICs or not {% if ARCH_CYC == cycle_HH | int %} -# Save the forecast-only cycle ICs every ARCH_WARMICFREQ or ARCH_FCSTICFREQ days -{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} -{% filter indent(width=4) %} + # Save the forecast-only cycle ICs every ARCH_WARMICFREQ or ARCH_FCSTICFREQ days + {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} + {% filter indent(width=4) %} {% include "gfs_restarta.yaml.j2" %} -{% endfilter %} -{% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} -{% filter indent(width=4) %} + {% endfilter %} + {% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} + {% filter indent(width=4) %} {% include "gfs_restarta.yaml.j2" %} -{% endfilter %} -{% endif %} + {% endfilter %} + {% endif %} {% endif %}