From 1ec9c1b2a87d3b8c87d5131e28928a91e30f2b21 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Fri, 21 Jun 2024 13:44:29 -0400 Subject: [PATCH] Fix and simplify online archiving and reenable METplus jobs (#2687) This fixes the online archiving portion of the `*arch` and `*earc00` jobs, a prerequisite for running METplus. This also reenables METplus by default. The approach previously taken created `FileHandler` dictionaries at varying levels within the resulting yaml, which was not properly parsed by `exglobal_archive.py`. This approach creates a single `FileHandler` dictionary and is much less complicated overall. Resolves #2673 #2647 --- .gitignore | 2 + parm/archive/arcdir.yaml.j2 | 250 ++++++++++++++++------------- parm/archive/gdas.yaml.j2 | 123 +++++++------- parm/config/gfs/yaml/defaults.yaml | 2 +- ush/python/pygfs/task/archive.py | 24 +-- 5 files changed, 215 insertions(+), 186 deletions(-) diff --git a/.gitignore b/.gitignore index 0c15a50661..861346a494 100644 --- a/.gitignore +++ b/.gitignore @@ -184,6 +184,8 @@ ush/fv3gfs_make_grid.sh ush/fv3gfs_make_orog.sh ush/gen_bufr2ioda_json.py ush/gen_bufr2ioda_yaml.py +ush/bufr2ioda_insitu_profile*.py +ush/bufr2ioda_insitu_surface*.py ush/global_chgres.sh ush/global_chgres_driver.sh ush/global_cycle.sh diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index d6b1899ebc..f845e3c9cb 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -4,121 +4,153 @@ {% set head = RUN + ".t" + cycle_HH + "z." %} # Select data to store in the ARCDIR and VFYARC from deterministic runs -Base: &base # GDAS, GFS, ENKFGDAS, or ENKFGFS - common: - mkdir: - - "{{ ARCDIR }}" - -# Common files to be added to both the gfs and gdas keys below -Deterministic: &deterministic - cyclone: - copy: - # Cyclone forecasts, produced for both gdas and gfs cycles - ## Only created if tracking is on and there were systems to track - {% if path_exists(COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} - - ["{{ COMIN_ATMOS_TRACK }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COMIN_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"] - {% endif %} - - # Cyclone tracking data - {% for basin in ["epac", "natl"] %} - {% if path_exists(COMIN_ATMOS_TRACK + "/" + basin) %} - - ["{{ COMIN_ATMOS_TRACK }}/{{ basin }}", "{{ ARCDIR }}/{{ basin }}"] - {% endif %} - {% endfor %} +# This file set will contain all source-destination pairs to send to the FileHandler for copying +{% set file_set = [] %} + +# Declare the VFYARC where Fit2Obs data will be sent +{% set VFYARC = ROTDIR ~ "/vrfyarch" %} + +# Deterministic files +{% if "enkf" not in RUN %} + # Common files to be added to both the gfs and gdas keys below + {% set det_files = [] %} + # Cyclone forecasts, produced for both gdas and gfs cycles + ## Only created if tracking is on and there were systems to track + {% if path_exists(COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} + {% do det_files.append([COMIN_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH, + ARCDIR ~"/atcfunix." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% do det_files.append([COMIN_ATMOS_TRACK ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH, + ARCDIR ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH]) %} + {% endif %} - {% if MODE == "cycled" %} - analysis: - copy: - # Analysis data (if we are running in cycled mode) - - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] - - {% if DO_JEDIATMVAR %} - - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}atmstat", "{{ ARCDIR }}/atmstat.{{ RUN }}.{{ cycle_YMDH }}"] - {% else %} - - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] - {% endif %} - - {% if DO_JEDISNOWDA %} - - ["{{ COMIN_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] - {% endif %} - - {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %} - - ["{{ COMIN_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] - {% endif %} - - {% if DO_PREP_OBS_AERO %} - - ["{{ COMIN_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COMIN_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"] - {% endif %} + # Cyclone tracking data + {% for basin in ["epac", "natl"] %} + {% if path_exists(COMIN_ATMOS_TRACK + "/" + basin) %} + {% do det_files.append([COMIN_ATMOS_TRACK ~ "/" ~ basin, + ARCDIR ~ "/" ~ basin ]) %} + {% endif %} + {% endfor %} + + # Deterministic analysis files (generated for cycled experiments) + {% set det_anl_files = [] %} + # Analysis data (if we are running in cycled mode) + {% do det_anl_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.anl", + ARCDIR ~ "/pgbanl." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} + + {% if DO_JEDIATMVAR == True %} + {% do det_anl_files.append([COMIN_ATMOS_ANALYSIS ~ "/" ~ head ~ "atmstat", + ARCDIR ~ "/atmstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% else %} + {% do det_anl_files.append([COMIN_ATMOS_ANALYSIS ~ "/" ~ head ~ "gsistat", + ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% endif %} + {% if DO_JEDISNOWDA == True %} + {% do det_anl_files.append([COMIN_SNOW_ANALYSIS ~ "/" ~ head ~ "snowstat.tgz", + ARCDIR ~ "/snowstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".tgz"]) %} {% endif %} -{% if RUN == "gfs" %} -gfs: # GFS specific - <<: *base - <<: *deterministic - - gfs: - copy: - {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %} - - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] - {% endfor %} - - # Cyclone genesis data (only present if there are storms) - {% if path_exists(COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} - - ["{{ COMIN_ATMOS_GENESIS }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}"] - - ["{{ COMIN_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}"] - {% endif %} - - {% if path_exists(COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} - - ["{{ COMIN_ATMOS_GENESIS }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"] - - ["{{ COMIN_ATMOS_GENESIS }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"] - {% endif %} - - {% if DO_FIT2OBS %} - fit2obs: - - mkdir: - {% set VFYARC = ROTDIR + "/vrfyarch" %} - - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" - - copy: - {% for fhr in range(0, FHMAX_FITS + 1, 6) %} - {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} - {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} - - ["{{COMIN_ATMOS_HISTORY}}/{{ sfcfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"] - - ["{{COMIN_ATMOS_HISTORY}}/{{ sigfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"] - {% endfor %} + {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %} + {% do det_anl_files.append([COMIN_CHEM_ANALYSIS ~ "/" ~ head ~ "aerostat", + ARCDIR ~ "/aerostat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% endif %} + {% if DO_PREP_OBS_AERO == True %} + {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroobs", + ARCDIR ~ "/aeroobs." ~ RUN ~ "." ~ cycle_YMDH]) %} + {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroawobs", + ARCDIR ~ "/aeroawobs." ~ RUN ~ "." ~ cycle_YMDH]) %} {% endif %} -{% endif %} -{% if RUN == "gdas" %} -gdas: # GDAS specific - <<: *base - <<: *deterministic - gdas: - copy: - {% for fhr in range(0, FHMAX + 1, FHOUT) %} - - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] - {% endfor %} -{% endif %} + # GFS-specific files + {% set gfs_files = [] %} + {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %} + {% do gfs_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr), + ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} + {% endfor %} + + # Cyclone genesis data (only present if there are storms) + {% if path_exists(COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH, + ARCDIR ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH ]) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH, + ARCDIR ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH ]) %} + {% endif %} + + {% if path_exists(COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH, + ARCDIR ~ "/trak.gfso.atcfunix." ~ cycle_YMDH ]) %} + {% do gfs_files.append([COMIN_ATMOS_GENESIS ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH, + ARCDIR ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH ]) %} + {% endif %} + + # GFS Fit2Obs data + {% set fit2obs_files = [] %} + {% for fhr in range(0, FHMAX_FITS + 1, 6) %} + {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} + {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} + {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sfcfile, + VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sfcfile ]) %} + {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sigfile, + VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sigfile ]) %} + {% endfor %} + + # GDAS-specific files + {% set gdas_files = [] %} + {% for fhr in range(0, FHMAX + 1, FHOUT) %} + {% do gdas_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr), + ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %} + {% endfor %} + + # Now append the necessary file pairs to file_set + # Common deterministic files + {% set file_set = file_set + det_files %} + {% if MODE == "cycled" %} + {% set file_set = file_set + det_anl_files %} + {% endif %} + + # Run-specific deterministic files + {% if RUN == "gfs" %} + {% set file_set = file_set + gfs_files %} + # Fit2Obs files + {% if DO_FIT2OBS == True %} + {% set file_set = file_set + fit2obs_files %} + {% endif %} + {% elif RUN == "gdas" %} + {% set file_set = file_set + gdas_files %} + {% endif %} + +{% else %} # End of deterministic files + + # Ensemble analysis files + {% set enkf_files = [] %} + {% if DO_JEDIATMENS == True %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atmensstat", + ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atminc.ensmean.nc", + ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean.nc"]) %} + {% else %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat", + ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean", + ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %} + {% endif %} + + # Construct the final file set + {% set file_set = file_set + enkf_files %} -Ensemble: &ensemble # ENKFGDAS or ENKFGFS - analysis: - copy: - # Copy ensemble analyses - {% if DO_JEDIATMENS %} - - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atmensstat", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atminc.ensmean.nc", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}.ensmean.nc"] - {% else %} - - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat", "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"] - - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat.ensmean", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"] - {% endif %} - -{% if "enkf" in RUN %} -{{ RUN }}: - <<: *base - <<: *ensemble {% endif %} + + +# Actually write the yaml +mkdir: + - "{{ ARCDIR }}" + + {% if DO_FIT2OBS == True %} + - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" + {% endif %} + +copy: + {% for source_dest_pair in file_set %} + - {{ source_dest_pair }} + {% endfor %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 12b89f2e61..b253d27268 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -7,32 +7,32 @@ gdas: # Cycled logs - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" - {% if DO_JEDIATMVAR %} + {% if DO_JEDIATMVAR %} - "logs/{{ cycle_YMDH }}/{{ RUN }}prepatmiodaobs.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlinit.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlfinal.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlfv3inc.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlvar.log" - {% else %} + {% else %} - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}analdiag.log" - {% endif %} + {% endif %} - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" - {% if DO_JEDIOCNVAR %} + {% if DO_JEDIOCNVAR %} - "logs/{{ cycle_YMDH }}/{{ RUN }}prepoceanobs.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalprep.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalbmat.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalrun.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalpost.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalchkpt.log" - {% if DOHYBVAR %} + {% if DOHYBVAR %} - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalecen.log" - {% endif %} - {% endif %} - {% if DO_VRFY_OCEANDA %} + {% endif %} + {% endif %} + {% if DO_VRFY_OCEANDA %} - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalvrfy.log" - {% endif %} + {% endif %} # Analysis GRIB2 (sub-sampled) data - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" @@ -44,117 +44,120 @@ gdas: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - {% if DOHYBVAR %} + {% if DOHYBVAR %} # Ensemble analysis residual - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" - {% if DOIAU %} + {% if DOIAU %} # Ensemble IAU analysis residuals - {% for fhr in iaufhrs if fhr != 6 %} + {% for fhr in iaufhrs if fhr != 6 %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensres.nc" - {% endfor %} - {% endif %} + {% endfor %} + {% endif %} # End of ensemble analysis mean residuals - {% endif %} + {% endif %} # Analysis state - {% if DO_JEDIATMVAR %} + {% if DO_JEDIATMVAR %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" - {% else %} + {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" - {% endif %} - {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} + {% endif %} + {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" - {% endif %} - {% if DO_PREP_OBS_AERO %} + {% endif %} + {% if DO_PREP_OBS_AERO %} - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" - {% endif %} - {% if DO_JEDISNOWDA %} + {% endif %} + {% if DO_JEDISNOWDA %} - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" - {% endif %} + {% endif %} # Ozone verification - {% if DO_VERFOZN %} + {% if DO_VERFOZN %} - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" - {% endif %} + {% endif %} # Radiance verification - {% if DO_VERFRAD %} + {% if DO_VERFRAD %} - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" - {% endif %} + {% endif %} # Minimization monitor - {% if DO_VMINMON %} + {% if DO_VMINMON %} - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" - {% endif %} + {% endif %} {% endif %} # End of cycled data # Forecast and post logs - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" - # TODO explicitly name the atmos_prod log files to archive - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - {% if not WRITE_DOPOST %} - # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" - {% endif %} ## not WRITE_DOPOST {% for fhr in range(0, FHMAX + 1, 3) %} + {% set fhr3 = '%03d' % fhr %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f{{ fhr3 }}.log" + {% if not WRITE_DOPOST %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f{{ fhr3 }}.log" + {% endif %} ## not WRITE_DOPOST # Forecast GRIB2 data - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ fhr3 }}" + - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ fhr3 }}.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ fhr3 }}" + - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ fhr3 }}.idx" # Forecast GRIB2 fluxes - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ fhr3 }}.grib2.idx" # FV3 log - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ fhr3 }}.txt" # Raw netCDF forecasts - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ fhr3 }}.nc" + - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ fhr3 }}.nc" {% endfor %} optional: {% if MODE == "cycled" %} - {% if DO_VERFRAD %} - # Radiance verification (only created if there are problems) + # Radiance verification (only created if there are problems) + {% if DO_VERFRAD %} - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" - {% endif %} + {% endif %} - {% if DO_VERFOZN %} - # Not all of these ozone instruments always produce data - {% set oznmon_types = [ - "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", - "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" - ] %} - {% for group in [ "horiz", "time" ] %} - {% if group == "horiz" %} {% set suffix = ".gz" %} {% else %} {% set suffix = "" %} {% endif %} - {% for type in oznmon_types %} + {% if DO_VERFOZN %} + # Not all of these ozone instruments always produce data + {% set oznmon_types = [ + "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", + "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" + ] %} + {% for group in [ "horiz", "time" ] %} + {% if group == "horiz" %} + {% set suffix = ".gz" %} + {% else %} + {% set suffix = "" %} + {% endif %} + {% for type in oznmon_types %} - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - {% endfor %} - {% endfor %} - {% endif %} + {% endfor %} + {% endfor %} + {% endif %} {% endif %} diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index b074b11dbf..2d662a9bcb 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -13,7 +13,7 @@ base: DO_TRACKER: "YES" DO_GENESIS: "YES" DO_GENESIS_FSU: "NO" - DO_METP: "NO" + DO_METP: "YES" FHMAX_GFS: 120 FHMAX_HF_GFS: 0 DO_VRFY_OCEANDA: "NO" diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 269aea0ccc..d0722552e1 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -7,9 +7,9 @@ from logging import getLogger from typing import Any, Dict, List -from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, cast_strdict_as_dtypedict, +from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, strftime, - to_YMD, to_YMDH, Template, TemplateConstants) + to_YMDH) logger = getLogger(__name__.split('.')[-1]) @@ -348,19 +348,11 @@ def _construct_arcdir_set(arcdir_j2yaml, arch_dict) -> Dict: files need to be copied to the ARCDIR and the Fit2Obs directory. """ - # Parse the input jinja yaml template - arcdir_yaml = parse_j2yaml(arcdir_j2yaml, - arch_dict, - allow_missing=True) - - # Collect the needed FileHandler dicts and construct arcdir_set - arcdir_set = {} - for key, handler in arcdir_yaml[arch_dict.RUN].items(): - # Different RUNs can have different filesets that need to be copied. - # Each fileset is stored as a dictionary. Collect the contents of - # each (which should be 'mkdir' and/or 'copy') to produce singular - # mkdir and copy lists. - arcdir_set.update(handler) + # Get the FileHandler dictionary for creating directories and copying + # to the ARCDIR and VFYARC directories. + arcdir_set = parse_j2yaml(arcdir_j2yaml, + arch_dict, + allow_missing=True) return arcdir_set @@ -421,7 +413,7 @@ def replace_string_from_to_file(filename_in, filename_out, search_str, replace_s with open(filename_in) as old_file: lines = old_file.readlines() - out_lines = [line.replace(search, replace) for line in lines] + out_lines = [line.replace(search_str, replace_str) for line in lines] with open("/tmp/track_file", "w") as new_file: new_file.writelines(out_lines)