Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix GDAS group B restart archiving #2735

Merged
merged 10 commits into from
Jul 9, 2024
Merged

Fix GDAS group B restart archiving #2735

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
576c874
Introduce tabbing to master archive templates
DavidHuber-NOAA Jun 28, 2024
04a355a
Save gdas restartb tarballs at a 6-hour offset from restarta
DavidHuber-NOAA Jun 28, 2024
18d0d92
Correct 'include' tabbing
DavidHuber-NOAA Jul 1, 2024
0280d2d
Merge remote-tracking branch 'origin/develop' into fix/restartb_archi…
DavidHuber-NOAA Jul 1, 2024
62aeca5
Merge remote-tracking branch 'origin/develop' into fix/restartb_archi…
DavidHuber-NOAA Jul 3, 2024
3ece3aa
Merge remote-tracking branch 'origin/develop' into fix/restartb_archi…
DavidHuber-NOAA Jul 3, 2024
593efa3
Separate warm/forecast-only restart ARCH_CYC calculations
DavidHuber-NOAA Jul 5, 2024
6e2fff0
Merge remote-tracking branch 'origin/develop' into fix/restartb_archi…
DavidHuber-NOAA Jul 5, 2024
c8306c3
Merge branch 'develop' into fix/restartb_archiving
DavidHuber-NOAA Jul 8, 2024
95ffc77
Merge branch 'develop' into fix/restartb_archiving
DavidHuber-NOAA Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 87 additions & 84 deletions parm/archive/master_enkf.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,108 +6,111 @@

# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
{% if IAUFHRS is string %}
# "3,6,9"
{% set iaufhrs = [] %}
{% for iaufhr in IAUFHRS.split(",") %}
{% do iaufhrs.append(iaufhr | int) %}
{% endfor %}
# "3,6,9"
{% set iaufhrs = [] %}
{% for iaufhr in IAUFHRS.split(",") %}
{% do iaufhrs.append(iaufhr | int) %}
{% endfor %}
{% else %}
# 6 (integer)
{% set iaufhrs = [IAUFHRS] %}
# 6 (integer)
{% set iaufhrs = [IAUFHRS] %}
{% endif %}

# Repeat for IAUFHRS_ENKF
{% if IAUFHRS_ENKF is string %}
{% set iaufhrs_enkf = [] %}
{% for iaufhr in IAUFHRS_ENKF.split(",") %}
{% do iaufhrs_enkf.append(iaufhr | int) %}
{% endfor %}
{% set iaufhrs_enkf = [] %}
{% for iaufhr in IAUFHRS_ENKF.split(",") %}
{% do iaufhrs_enkf.append(iaufhr | int) %}
{% endfor %}
{% else %}
{% set iaufhrs_enkf = [IAUFHRS_ENKF] %}
{% set iaufhrs_enkf = [IAUFHRS_ENKF] %}
{% endif %}

# Determine which data to archive
datasets:
{% if ENSGRP == 0 %}
{% filter indent(width=4) %}
# Archive the ensemble means and spreads
{% filter indent(width=4) %}
# Archive the ensemble means and spreads
{% include "enkf.yaml.j2" %}
{% endfilter %}
{% endfilter %}
{% else %}

# Archive individual member data
# First, construct individual member directories from templates
# COMIN_ATMOS_ANALYSIS_MEM, COMIN_ATMOS_HISTORY_MEM, and COMIN_ATMOS_RESTART_MEM

# Declare to-be-filled lists of member COM directories
{% set COMIN_ATMOS_ANALYSIS_MEM_list = [] %}
{% set COMIN_ATMOS_RESTART_MEM_list = [] %}
{% set COMIN_ATMOS_HISTORY_MEM_list = [] %}

# Determine which ensemble members belong to this group
{% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %}
{% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, nmem_ens ] | min %}

# Construct member COM directories for the group
{% for mem in range(first_group_mem, last_group_mem + 1) %}

# Declare a dict of search and replace terms to run on each template
{% set tmpl_dict = {'ROTDIR':ROTDIR,
'RUN':RUN,
'YMD':cycle_YMD,
'HH':cycle_HH,
'MEMDIR':"mem" + '%03d' % mem} %}

# Replace template variables with tmpl_dict, one key at a time
# This must be done in a namespace to overcome jinja scoping
# Variables set inside of a for loop are lost at the end of the loop
# unless they are part of a namespace
{% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL,
COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL,
COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %}

{% for key in tmpl_dict.keys() %}
{% set search_term = '${' + key + '}' %}
{% set replace_term = tmpl_dict[key] %}
{% set com_ns.COMIN_ATMOS_ANALYSIS_MEM = com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %}
{% set com_ns.COMIN_ATMOS_HISTORY_MEM = com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %}
{% set com_ns.COMIN_ATMOS_RESTART_MEM = com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %}
{% endfor %}

# Append the member COM directories
{% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%}
{% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%}
{% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%}

{% endfor %}

# Archive member data
{% filter indent(width=4) %}
# Archive individual member data
# First, construct individual member directories from templates
# COMIN_ATMOS_ANALYSIS_MEM, COMIN_ATMOS_HISTORY_MEM, and COMIN_ATMOS_RESTART_MEM

# Declare to-be-filled lists of member COM directories
{% set COMIN_ATMOS_ANALYSIS_MEM_list = [] %}
{% set COMIN_ATMOS_RESTART_MEM_list = [] %}
{% set COMIN_ATMOS_HISTORY_MEM_list = [] %}

# Determine which ensemble members belong to this group
{% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %}
{% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, nmem_ens ] | min %}

# Construct member COM directories for the group
{% for mem in range(first_group_mem, last_group_mem + 1) %}

# Declare a dict of search and replace terms to run on each template
{% set tmpl_dict = {'ROTDIR':ROTDIR,
'RUN':RUN,
'YMD':cycle_YMD,
'HH':cycle_HH,
'MEMDIR':"mem" + '%03d' % mem} %}

# Replace template variables with tmpl_dict, one key at a time
# This must be done in a namespace to overcome jinja scoping
# Variables set inside of a for loop are lost at the end of the loop
# unless they are part of a namespace
{% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL,
COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL,
COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %}

{% for key in tmpl_dict.keys() %}
{% set search_term = '${' + key + '}' %}
{% set replace_term = tmpl_dict[key] %}
{% set com_ns.COMIN_ATMOS_ANALYSIS_MEM =
com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %}
{% set com_ns.COMIN_ATMOS_HISTORY_MEM =
com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %}
{% set com_ns.COMIN_ATMOS_RESTART_MEM =
com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %}
{% endfor %}

# Append the member COM directories
{% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%}
{% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%}
{% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%}

{% endfor %}

# Archive member data
{% filter indent(width=4) %}
{% include "enkf_grp.yaml.j2" %}
{% endfilter %}
{% endfilter %}

# Determine if restarts should be saved
{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %}
# Determine if restarts should be saved
{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %}

# Save the increments and restarts every ARCH_WARMICFREQ days
# The ensemble increments (group a) should be saved on the ARCH_CYC
{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %}
{% if ARCH_CYC == cycle_HH | int %}
{% filter indent(width=4) %}
# Save the increments and restarts every ARCH_WARMICFREQ days
# The ensemble increments (group a) should be saved on the ARCH_CYC
{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %}
{% if ARCH_CYC == cycle_HH | int %}
{% filter indent(width=4) %}
{% include "enkf_restarta_grp.yaml.j2" %}
{% endfilter %}
{% endif %}
{% endif %}

# The ensemble ICs (group b) are restarts and always lag increments by assim_freq
{% set ics_offset = (assim_freq | string + "H") | to_timedelta %}
{% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 %}
{% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int %}
{% filter indent(width=4) %}
{% endfilter %}
{% endif %}
{% endif %}

# The ensemble ICs (group b) are restarts and always lag increments by assim_freq
{% set ics_offset = (assim_freq | string + "H") | to_timedelta %}
{% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 %}
{% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int %}
{% filter indent(width=4) %}
{% include "enkf_restartb_grp.yaml.j2" %}
{% endfilter %}
{% endif %}
{% endif %}
{% endfilter %}
{% endif %}
{% endif %}

# End of individual member archiving
# End of individual member archiving
{% endif %}
124 changes: 65 additions & 59 deletions parm/archive/master_gdas.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@

# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
{% if IAUFHRS is string %}
{% set iaufhrs = [] %}
{% for iaufhr in IAUFHRS.split(",") %}
{% do iaufhrs.append(iaufhr | int) %}
{% endfor %}
{% set iaufhrs = [] %}
{% for iaufhr in IAUFHRS.split(",") %}
{% do iaufhrs.append(iaufhr | int) %}
{% endfor %}
{% else %}
{% set iaufhrs = [IAUFHRS] %}
{% set iaufhrs = [IAUFHRS] %}
{% endif %}

datasets:
Expand All @@ -20,84 +20,90 @@ datasets:
{% endfilter %}

{% if DO_ICE %}
# Ice data
{% filter indent(width=4) %}
# Ice data
{% filter indent(width=4) %}
{% include "gdasice.yaml.j2" %}
{% endfilter %}
{% endfilter %}
{% endif %}

{% if DO_OCN %}
# Ocean forecast products
{% filter indent(width=4) %}
# Ocean forecast products
{% filter indent(width=4) %}
{% include "gdasocean.yaml.j2" %}
{% endfilter %}
{% if DO_JEDIOCNVAR and MODE == "cycled" %}
# Ocean analysis products
{% filter indent(width=4) %}
{% endfilter %}
{% if DO_JEDIOCNVAR and MODE == "cycled" %}
# Ocean analysis products
{% filter indent(width=4) %}
{% include "gdasocean_analysis.yaml.j2" %}
{% endfilter %}
{% endif %}
{% endfilter %}
{% endif %}
{% endif %}

{% if DO_WAVE %}
# Wave products
{% filter indent(width=4) %}
# Wave products
{% filter indent(width=4) %}
{% include "gdaswave.yaml.j2" %}
{% endfilter %}
{% endfilter %}
{% endif %}

{% if MODE == "cycled" %}
# Determine if we will save restart ICs or not (only valid for cycled)
{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %}
# Determine if we will save restart ICs or not (only valid for cycled)
{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %}

{% if ARCH_CYC == cycle_HH | int%}
# Save the warm and forecast-only cycle ICs every ARCH_WARMICFREQ days
{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %}
{% set save_warm_start_forecast = True %}
{% set save_warm_start_cycled = True %}
# Save the forecast-only restarts every ARCH_FCSTICFREQ days
{% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %}
{% set save_warm_start_forecast = True %}
{% endif %}
{% endif %}
{% if ARCH_CYC == cycle_HH | int%}
# Save the forecast-only cycle ICs every ARCH_WARMICFREQ or ARCH_FCSTICFREQ days
{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %}
{% set save_warm_start_forecast = True %}
{% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %}
{% set save_warm_start_forecast = True %}
{% endif %}
{% endif %}

{% if save_warm_start_forecast %}
# Save warm start forecast-only data
# Atmosphere restarts
{% filter indent(width=4) %}
# The GDAS ICs (group b) are restarts and always lag increments by assim_freq
{% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int %}
{% set ics_offset = (assim_freq | string + "H") | to_timedelta %}
{% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 %}
{% set save_warm_start_cycled = True %}
{% endif %}
{% endif %}

{% if save_warm_start_forecast %}
# Save warm start forecast-only data
# Atmosphere restarts
{% filter indent(width=4) %}
{% include "gdas_restarta.yaml.j2" %}
{% endfilter %}
{% endfilter %}

{% if DO_WAVE %}
# Wave restarts
{% filter indent(width=4) %}
{% if DO_WAVE %}
# Wave restarts
{% filter indent(width=4) %}
{% include "gdaswave_restart.yaml.j2" %}
{% endfilter %}
{% endif %}
{% endfilter %}
{% endif %}

{% if DO_OCN %}
# Ocean restarts
{% filter indent(width=4) %}
{% if DO_OCN %}
# Ocean restarts
{% filter indent(width=4) %}
{% include "gdasocean_restart.yaml.j2" %}
{% endfilter %}
{% endif %}
{% endfilter %}
{% endif %}

{% if DO_ICE %}
# Ice restarts
{% filter indent(width=4) %}
{% if DO_ICE %}
# Ice restarts
{% filter indent(width=4) %}
{% include "gdasice_restart.yaml.j2" %}
{% endfilter %}
{% endif %}
{% endfilter %}
{% endif %}

# End of forecast-only restarts
{% endif %}
# End of forecast-only restarts
{% endif %}

{% if save_warm_start_cycled %}
# Save warm start cycled restarts
{% filter indent(width=4) %}
{% if save_warm_start_cycled %}
# Save warm start cycled restarts
{% filter indent(width=4) %}
{% include "gdas_restartb.yaml.j2" %}
{% endfilter %}
{% endif %}
{% endfilter %}
{% endif %}

# End of restart checking
# End of restart checking
{% endif %}
Loading
Loading