c48 atm forecast only on google

env/AZUREPW.env

@@ -0,0 +1,87 @@
+#! /usr/bin/env bash
+
+if [[ $# -ne 1 ]]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    exit 1
+
+fi
+
+step=$1
+
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
+
+# Configure MPI environment
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+
+ulimit -s unlimited
+ulimit -a
+
+# Calculate common variables
+# Check first if the dependent variables are set
+if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
+    max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
+    NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
+    NTHREADS1=${threads_per_task:-1}
+    [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
+    [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
+    APRUN="${launcher} -n ${ntasks}"
+else
+    echo "ERROR config.resources must be sourced before sourcing AZUREPW.env"
+    exit 2
+fi
+
+if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+
+    export launcher="srun --mpi=pmi2 -l"
+
+    (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
+    (( ufs_ntasks = nnodes*tasks_per_node ))
+    # With ESMF threading, the model wants to use the full node
+    export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
+    unset nnodes ufs_ntasks
+
+elif [[ "${step}" = "post" ]]; then
+
+    export NTHREADS_NP=${NTHREADS1}
+    export APRUN_NP="${APRUN}"
+
+    export NTHREADS_DWN=${threads_per_task_dwn:-1}
+    [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
+    export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    export NTHREADS_ECEN=${NTHREADSmax}
+    export APRUN_ECEN="${APRUN}"
+
+    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${APRUN}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    export NTHREADS_ESFC=${NTHREADSmax}
+    export APRUN_ESFC="${APRUN}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    export APRUN_CYCLE="${APRUN}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    export NTHREADS_EPOS=${NTHREADSmax}
+    export APRUN_EPOS="${APRUN}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN}"
+
+fi

env/GOOGLEPW.env

@@ -0,0 +1,87 @@
+#! /usr/bin/env bash
+
+if [[ $# -ne 1 ]]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    exit 1
+
+fi
+
+step=$1
+
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
+
+# Configure MPI environment
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+
+ulimit -s unlimited
+ulimit -a
+
+# Calculate common variables
+# Check first if the dependent variables are set
+if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
+    max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
+    NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
+    NTHREADS1=${threads_per_task:-1}
+    [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
+    [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
+    APRUN="${launcher} -n ${ntasks}"
+else
+    echo "ERROR config.resources must be sourced before sourcing GOOGLEPW.env"
+    exit 2
+fi
+
+if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+
+    export launcher="srun --mpi=pmi2 -l"
+
+    (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
+    (( ufs_ntasks = nnodes*tasks_per_node ))
+    # With ESMF threading, the model wants to use the full node
+    export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
+    unset nnodes ufs_ntasks
+
+elif [[ "${step}" = "post" ]]; then
+
+    export NTHREADS_NP=${NTHREADS1}
+    export APRUN_NP="${APRUN}"
+
+    export NTHREADS_DWN=${threads_per_task_dwn:-1}
+    [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
+    export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    export NTHREADS_ECEN=${NTHREADSmax}
+    export APRUN_ECEN="${APRUN}"
+
+    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${APRUN}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    export NTHREADS_ESFC=${NTHREADSmax}
+    export APRUN_ESFC="${APRUN}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    export APRUN_CYCLE="${APRUN}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    export NTHREADS_EPOS=${NTHREADSmax}
+    export APRUN_EPOS="${APRUN}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN}"
+
+fi

parm/config/gfs/config.resources

@@ -112,6 +112,22 @@ case ${machine} in
     # shellcheck disable=SC2034
     mem_node_max=""
     ;;
+  "AZUREPW")
+    export PARTITION_BATCH="compute"
+    npe_node_max=24
+    max_tasks_per_node=24
+    # TODO Supply a max mem/node value for AZURE
+    # shellcheck disable=SC2034
+    mem_node_max=""
+    ;;
+  "GOOGLEPW")
+    export PARTITION_BATCH="compute"
+    npe_node_max=30
+    max_tasks_per_node=30
+    # TODO Supply a max mem/node value for GOOGLE
+    # shellcheck disable=SC2034
+    mem_node_max=""
+    ;;
   "CONTAINER")
     max_tasks_per_node=1
     # TODO Supply a max mem/node value for a container

parm/config/gfs/config.resources.AZUREPW

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+# AWS-specific job resources
+
+export is_exclusive="True"
+export memory=None
+
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done

parm/config/gfs/config.resources.GOOGLEPW

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+# GOOGLE-specific job resources
+
+export is_exclusive="True"
+export memory=None
+
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done

workflow/hosts.py

@@ -16,7 +16,8 @@ class Host:
     """
 
     SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES',
-                       'WCOSS2', 'S4', 'CONTAINER', 'AWSPW', 'GAEA']
+                       'WCOSS2', 'S4', 'CONTAINER', 'GAEA',
+                       'AWSPW', 'AZUREPW', 'GOOGLEPW']
 
     def __init__(self, host=None):
 

workflow/hosts/azurepw.yaml

@@ -0,0 +1,26 @@
+BASE_GIT: '' #TODO: This does not yet exist.
+DMPDIR: '' # TODO: This does not yet exist.
+PACKAGEROOT: '' #TODO: This does not yet exist.
+COMINsyn: '' #TODO: This does not yet exist.
+HOMEDIR: '/contrib/${USER}'
+STMP: '/lustre/${USER}/stmp/'
+PTMP: '/lustre/${USER}/ptmp/'
+NOSCRUB: '${HOMEDIR}'
+ACCOUNT: '${USER}'
+SCHEDULER: slurm
+QUEUE: batch
+QUEUE_SERVICE: batch
+PARTITION_BATCH: compute
+PARTITION_SERVICE: compute
+RESERVATION: ''
+CLUSTERS: ''
+CHGRP_RSTPROD: 'YES'
+CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
+HPSSARCH: 'NO'
+HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
+BASE_CPLIC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs'
+LOCALARCH: 'NO'
+ATARDIR: '' # TODO: This will not yet work from AWS.
+MAKE_NSSTBUFR: 'NO'
+MAKE_ACFTBUFR: 'NO'
+SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions.

workflow/hosts/googlepw.yaml

@@ -0,0 +1,28 @@
+BASE_GIT: '' #TODO: This does not yet exist.
+DMPDIR: '' # TODO: This does not yet exist.
+PACKAGEROOT: '' #TODO: This does not yet exist.
+COMINsyn: '' #TODO: This does not yet exist.
+HOMEgfs: '/contrib/${USER}/src/global-workflow-cloud'
+HOMEDIR: '/contrib/${USER}'
+STMP: '/contrib/${USER}/stmp/'
+PTMP: '/contrib/${USER}/ptmp/'
+NOSCRUB: '${HOMEDIR}'
+ACCOUNT: '${USER}'
+SCHEDULER: slurm
+QUEUE: batch
+QUEUE_SERVICE: batch
+PARTITION_BATCH: compute
+PARTITION_SERVICE: compute
+RESERVATION: ''
+CLUSTERS: ''
+CHGRP_RSTPROD: 'YES'
+CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
+HPSSARCH: 'NO'
+HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
+#BASE_CPLIC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs'
+BASE_CPLIC: '/contrib/global-workflow-shared-data/ICDIRS/prototype_ICs'
+LOCALARCH: 'NO'
+ATARDIR: '' # TODO: This will not yet work from GOOGLE.
+MAKE_NSSTBUFR: 'NO'
+MAKE_ACFTBUFR: 'NO'
+SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions.

workflow/rocoto/rocoto.py

@@ -146,7 +146,8 @@ def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]:
     strings.append(f'\t<walltime>{walltime}</walltime>\n')
     strings.append(f'\t<nodes>{nodes}:ppn={ppn}:tpp={threads}</nodes>\n')
     if memory is not None:
-        strings.append(f'\t<memory>{memory}</memory>\n')
+        if memory != 'None':
+            strings.append(f'\t<memory>{memory}</memory>\n')
     if native is not None:
         strings.append(f'\t<native>{native}</native>\n')
     strings.append('\n')