Cycling on native grid and create JEDI-based analysis calc job

env/HERA.env

@@ -160,6 +160,11 @@ elif [[ "${step}" = "marineanalletkf" ]]; then
     export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
     export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
 
+elif [[ "${step}" = "calcanl" ]]; then
+
+    export NTHREADS_CALCANL=${NTHREADSmax}
+    export APRUN_CALCANL="${APRUN_default} --cpus-per-task=${NTHREADS_CALCANL}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/HERCULES.env

@@ -154,7 +154,12 @@ case ${step} in
 
     export NTHREADS_OCNANAL=${NTHREADSmax}
     export APRUN_OCNANAL="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
- ;;
+;;
+ "calcanl")
+
+    export NTHREADS_CALCANL=${NTHREADSmax}
+    export APRUN_CALCANL="${APRUN_default} --cpus-per-task=${NTHREADS_CALCANL}"
+;; 
  "anal" | "analcalc")
 
     export MKL_NUM_THREADS=4

env/JET.env

@@ -126,6 +126,11 @@ elif [[ "${step}" = "ocnanalrun" ]]; then
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
     export APRUN_OCNANAL="${APRUN_default}"
 
+elif [[ "${step}" = "calcanl" ]]; then
+
+    export NTHREADS_CALCANL=${NTHREADSmax}
+    export APRUN_CALCANL="${APRUN_default}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/ORION.env

@@ -153,6 +153,11 @@ elif [[ "${step}" = "marineanalletkf" ]]; then
     export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
     export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
 
+elif [[ "${step}" = "calcanl" ]]; then
+
+    export NTHREADS_CALCANL=${NTHREADSmax}
+    export APRUN_CALCANL="${APRUN_default} --cpus-per-task=${NTHREADS_CALCANL}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/S4.env

@@ -122,6 +122,11 @@ elif [[ "${step}" = "marinebmat" ]]; then
 elif [[ "${step}" = "marinerun" ]]; then
      echo "WARNING: ${step} is not enabled on S4!"
 
+elif [[ "${step}" = "calcanl" ]]; then
+
+    export NTHREADS_CALCANL=${NTHREADSmax}
+    export APRUN_CALCANL="${APRUN_default}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/WCOSS2.env

@@ -136,6 +136,11 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN_default}"
 
+elif [[ "${step}" = "calcanl" ]]; then
+
+    export NTHREADS_CALCANL=${NTHREADSmax}
+    export APRUN_CALCANL="${APRUN_default}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export OMP_PLACES=cores

jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI

@@ -0,0 +1,55 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "calcanl" -c "base calcanl"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
+GDUMP="gdas"
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COM_ATMOS_ANALYSIS
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
+
+##############################################
+# Run relevant script
+##############################################
+
+EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_calc_analysis.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##############################################
+# Remove the Temporary working directory
+##############################################
+
+cd "${DATAROOT}" || ( echo "FATAL ERROR: ${DATAROOT} does not exist, ABORT!"; exit 1 )
+if [[ ${KEEPDATA} = "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

jobs/rocoto/analcalc_fv3jedi.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="calcanl"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_ATM_CALC_ANALYSIS"
+status=$?
+exit "${status}"

parm/config/gfs/config.analcalc_fv3jedi

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+########## config.calcanl ##########
+# Diagnostic amospheric analysis calculation specific
+
+echo "BEGIN: config.calcanl"
+
+export layout_x_calcanl=2
+export layout_y_calcanl=2
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" calcanl
+
+export JCB_BASE_YAML=${PARMgfs}/gdas/atm/jcb-base.yaml.j2
+export JCB_ALGO=fv3jedi_calcanl
+export JEDIEXE=${EXECgfs}/fv3jedi_calcanl.x
+
+if [[ ${DOHYBVAR} = "YES" ]]; then
+    export CASE_ANL=${CASE_ENS}
+else
+    export CASE_ANL=${CASE}
+fi
+
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
+echo "END: config.calcanl"

parm/config/gfs/config.resources

@@ -17,7 +17,7 @@ if (( $# != 1 )); then
     echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal"
     echo "snowanl esnowrecen"
     echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb"
-    echo "anal sfcanl analcalc analdiag fcst echgres"
+    echo "anal sfcanl calcanl analcalc analdiag fcst echgres"
     echo "upp atmos_products"
     echo "tracker genesis genesis_fsu"
     echo "verfozn verfrad vminmon fit2obs metp arch cleanup"
@@ -700,6 +700,17 @@ case ${step} in
     export is_exclusive=True
     ;;
 
+  "calcanl")
+    export layout_x=${layout_x_calcanl}
+    export layout_y=${layout_y_calcanl}
+
+    walltime="00:15:00"
+    ntasks=$(( layout_x * layout_y * 6 ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    export is_exclusive=True
+    ;;
+
   "analcalc")
     walltime="00:15:00"
     ntasks=127

scripts/exglobal_atmos_analysis_calc_fv3jedi.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# exglobal_atm_calc_analysis.py
+# This script creates an CalcAnalysis object
+# and runs the execute method which executes
+# the diagnostic global analysis calculation
+import os
+
+from wxflow import Logger, cast_strdict_as_dtypedict
+from pygfs.task.calcanl import CalcAnalysis
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config = cast_strdict_as_dtypedict(os.environ)
+
+    # Instantiate the CalcAnalysis task
+    CalcAnl = CalcAnalysis(config, 'calcanl')
+
+    # Initialize
+    CalcAnl.initialize_jedi()
+    CalcAnl.initialize()
+
+    # Execute JEDI application
+    CalcAnl.execute(config.APRUN_CALCANL)
+
+    # Finalize
+    CalcAnl.finalize()

sorc/link_workflow.sh

@@ -368,6 +368,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
                        "fv3jedi_plot_field.x" \
                        "gdasapp_chem_diagb.x" \
                        "fv3jedi_fv3inc.x" \
+                       "fv3jedi_calcanl.x" \
                        "gdas_ens_handler.x" \
                        "gdas_incr_handler.x" \
                        "gdas_obsprovider2ioda.x" \

ush/forecast_postdet.sh

@@ -95,6 +95,7 @@ FV3_postdet() {
         read_increment=".true."
         res_latlon_dynamics="atminc.nc"
       fi
+      increment_file_on_native_grid=".false."
       local increment_file
       for inc_file in "${inc_files[@]}"; do
         increment_file="${COMIN_ATMOS_INPUT}/${RUN}.t${cyc}z.${inc_file}"
@@ -158,9 +159,16 @@ EOF
           delimiter=","
         done
       else  # "${DOIAU}" == "NO"
-        inc_files=("atminc.nc")
         read_increment=".true."
-        res_latlon_dynamics="atminc.nc"
+	if [[ "${DO_JEDIATMVAR:-NO}" == "YES" ]]; then
+	  inc_files=("atminc.tile1.nc" "atminc.tile2.nc" "atminc.tile3.nc" "atminc.tile4.nc" "atminc.tile5.nc" "atminc.tile6.nc")
+          res_latlon_dynamics="atminc"
+	  increment_file_on_native_grid=".true."
+	else
+	  inc_files=("atminc.nc")
+          res_latlon_dynamics="atminc.nc"
+          increment_file_on_native_grid=".false."
+	fi    
         if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
           IAU_FHROT=${half_window}  # Replay ICs start at the end of the assimilation window
           # Control member has no perturbation
@@ -169,6 +177,7 @@ EOF
             read_increment=".false."
             res_latlon_dynamics='""'
           fi
+	  increment_file_on_native_grid=".false."
         fi
       fi
 

ush/forecast_predet.sh

@@ -362,6 +362,7 @@ FV3_predet(){
   warm_start=".false."
   read_increment=".false."
   res_latlon_dynamics='""'
+  increment_file_on_native_grid=".true."
 
   # Stochastic Physics Options
   do_skeb=".false."

ush/parsing_namelists_FV3.sh

@@ -180,6 +180,7 @@ cat > input.nml <<EOF
   agrid_vel_rst = ${agrid_vel_rst:-".true."}
   read_increment = ${read_increment}
   res_latlon_dynamics = ${res_latlon_dynamics}
+  increment_file_on_native_grid = ${increment_file_on_native_grid}
   ${fv_core_nml-}
 /
 

ush/python/pygfs/__init__.py

@@ -8,6 +8,7 @@
 from .task.aero_bmatrix import AerosolBMatrix
 from .task.atm_analysis import AtmAnalysis
 from .task.atmens_analysis import AtmEnsAnalysis
+from .task.calcanl import CalcAnalysis
 from .task.marine_bmat import MarineBMat
 from .task.snow_analysis import SnowAnalysis
 from .task.snowens_analysis import SnowEnsAnalysis