Merge branch 'NOAA-EMC:develop' into gw_ci_20240506

.gitignore

@@ -48,6 +48,8 @@ parm/gdas/io
 parm/gdas/ioda
 parm/gdas/snow
 parm/gdas/soca
+parm/gdas/jcb-gdas
+parm/gdas/jcb-algorithms
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
@@ -195,3 +197,8 @@ versions/run.ver
 ush/python/wxflow
 workflow/wxflow
 ci/scripts/wxflow
+
+# jcb checkout and symlinks
+ush/python/jcb
+workflow/jcb
+ci/scripts/jcb

.gitmodules

@@ -30,3 +30,7 @@
   path = sorc/upp.fd
   url = https://github.com/NOAA-EMC/UPP.git
   ignore = dirty
+[submodule "sorc/jcb"]
+  path = sorc/jcb
+  url = https://github.com/noaa-emc/jcb
+  fetchRecurseSubmodules = false

ci/Jenkinsfile

@@ -78,7 +78,7 @@ pipeline {
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
                         properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
-                        sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
+                        sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
                     }
                     echo "Building and running on ${Machine} in directory ${HOME}"
@@ -149,7 +149,7 @@ pipeline {
                                                 }
                                             }
                                             sh(script: './link_workflow.sh')
-                                            sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+                                            //sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
                                         }
                                     }
                                     if (env.CHANGE_ID && system == 'gfs') {

ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml

@@ -0,0 +1,18 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 384
+  resdetocean: 0.25
+  nens: 0
+  gfs_cyc: 4
+  start: cold
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 2021063000
+  edate: 2021070306
+  icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2
+  yaml: {{ HOMEgfs }}/ci/cases/gfsv17/ocnanal.yaml

ci/cases/gfsv17/ocnanal.yaml

@@ -0,0 +1,30 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+
+base:
+  DOIAU: "YES"
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "YES"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DOHYBVAR: "NO"
+  DO_FIT2OBS: "YES"
+  DO_VERFOZN: "YES"
+  DO_VERFRAD: "YES"
+  DO_VRFY_OCEANDA: "NO"
+  FHMAX_GFS: 240
+
+ocnanal:
+  SOCA_INPUT_FIX_DIR: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/1440x1080x75/soca
+  CASE_ANL: 'C24'
+  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
+  SOCA_NINNER: 100
+  SABER_BLOCKS_YAML: ''
+  NICAS_RESOL: 1
+  NICAS_GRID_SIZE: 15000
+
+prepoceanobs:
+  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml 
+  OBSPREP_YAML: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml
+  DMPDIR: /scratch1/NCEPDEV/da/common/

ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

@@ -1,6 +1,5 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  DOIAU: "NO"
   DO_JEDISNOWDA: "YES"
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}

ci/cases/yamls/gefs_ci_defaults.yaml

@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}

ci/cases/yamls/gfs_defaults_ci.yaml

@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}

ci/cases/yamls/gfs_extended_ci.yaml

@@ -2,7 +2,7 @@ defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
   DO_GOES: "YES"
   DO_BUFRSND: "YES"
   DO_GEMPAK: "YES"

ci/cases/yamls/soca_gfs_defaults_ci.yaml

@@ -1,5 +1,5 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
   DO_JEDIOCNVAR: "YES"

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -4,7 +4,7 @@ base:
   DOIAU: "NO"
   DO_JEDIATMVAR: "YES"
   DO_JEDIATMENS: "YES"
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 atmanl:
   LAYOUT_X_ATMANL: 4
   LAYOUT_Y_ATMANL: 4

ci/platforms/config.hera

@@ -2,7 +2,9 @@
 
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
-export STMP="/scratch1/NCEPDEV/stmp2/${USER}"
-export SLURM_ACCOUNT=nems
+export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4
+
+export JENKINS_AGENT_LANUCH_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/Jenkins
+export JENKINS_WORK_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness

ci/platforms/config.hercules

@@ -2,7 +2,9 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work2/noaa/stmp/${USER}"
-export SLURM_ACCOUNT=nems
+export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4
+
+export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry
+export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS

ci/platforms/config.orion

@@ -2,7 +2,9 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work2/noaa/stmp/${USER}"
-export SLURM_ACCOUNT=nems
+export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4
+
+export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry
+export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS

ci/platforms/config.wcoss2

@@ -2,7 +2,6 @@
 
 export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
 export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
-export STMP="/lfs/h2/emc/stmp/${USER}"
-export SLURM_ACCOUNT=GFS-DEV
+export HPC_ACCOUNT=GFS-DEV
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/scripts/driver.sh

@@ -220,18 +220,20 @@ for pr in ${pr_list}; do
       set +e
       export LOGFILE_PATH="${HOMEgfs}/ci/scripts/create_experiment.log"
       rm -f "${LOGFILE_PATH}"
+      yaml_case_file="${HOMEgfs}/ci/cases/pr/${case}.yaml"
+      skip_hosts=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" --yaml "${yaml_case_file}" --key skip_ci_on_hosts --string)
+      if [[ "${skip_hosts}" == *"${MACHINE_ID}"* ]]; then
+        {
+          echo "Case setup: Skipped for experiment ${pslot}" || true
+        } >> "${output_ci}"
+        continue
+      fi
       "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml"  > "${LOGFILE_PATH}" 2>&1
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then
-        last_line=$(tail -1 "${LOGFILE_PATH}")
-        if [[ "${last_line}" == *"Skipping creation"* ]]; then
-          action="Skipped"
-        else
-          action="Completed"
-        fi
         {
-          echo "Case setup: ${action} for experiment ${pslot}" || true
+          echo "Case setup: Completed for experiment ${pslot}" || true
         } >> "${output_ci}"
       else
         {

ci/scripts/run-check_ci.sh

@@ -64,7 +64,7 @@ while true; do
   rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
 
   # Wait before running rocotostat
-  sleep 10
+  sleep 60
 
   # Get job statistics
   echo "Gather Rocoto statistics"

ci/scripts/utils/ci_utils.sh

@@ -122,7 +122,8 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  # system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true
+  # Remove RUNDIRS dir incase this is a retry
+  rm -Rf "${STMP}/RUNDIRS/${pslot}"
 
   "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}"
 

ci/scripts/utils/launch_java_agent.sh

@@ -0,0 +1,77 @@
+#!/bin/env bash
+set -e
+
+controller_url="https://jenkins.epic.oarcloud.noaa.gov"
+controller_user="terry.mcguinness"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
+host=$(hostname)
+
+#########################################################################
+#  Set up runtime environment varibles for accounts on supproted machines
+#########################################################################
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+case ${MACHINE_ID} in
+  hera | orion | hercules | wcoss2)
+    echo "Launch Jenkins Java Controler on ${MACHINE_ID}"
+    ;;
+  *)
+    echo "Unsupported platform. Exiting with error."
+    exit 1
+    ;;
+esac
+
+LOG=lanuched_agent-$(date +%Y%m%d%M).log
+rm -f "${LOG}"
+
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/modulefiles"
+module load "module_gwsetup.${MACHINE_ID}"
+source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
+
+JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10"
+if [[ ! -d "${JAVA_HOME}" ]]; then
+  JAVA_HOME=/usr/lib/jvm/jre-17
+  if [[ ! -d "${JAVA_HOME}" ]]; then
+    echo "ERROR: JAVA_HOME not found. Exiting with error."
+    exit 1
+  fi
+fi
+
+JAVA="${JAVA_HOME}/bin/java"
+echo "JAVA VERSION: "
+${JAVA} -version
+
+export GH="${HOME}/bin/gh"
+command -v "${GH}"
+${GH} --version
+
+if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then
+  echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}"
+else
+  echo "ERROR: Jenkins Agent Lanuch Directory not found. Exiting with error."
+  exit 1
+fi
+cd "${JENKINS_AGENT_LANUCH_DIR}"
+
+if ! [[ -f agent.jar ]]; then
+  curl -sO "${controller_url}/jnlpJars/agent.jar"
+fi
+
+JENKINS_TOKEN=$(cat jenkins_token)
+
+# 
+offline=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') || true
+echo "Jenkins Agent offline setting: ${offline}"
+
+if [[ "${offline}" == "true" ]]; then
+  echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}"
+  command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp  -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}"
+  echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}"
+  ${command} >> "${LOG}" 2>&1 &
+  nohup_PID=$!
+  echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1
+  echo "Java agent running on PID: ${nohup_PID}"
+else
+  echo "Jenkins Agent is online (nothing done)"
+fi

ci/scripts/utils/parse_yaml.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+"""
+This script parses a yaml file and returns the value of a specified key.
+"""
+
+import os
+import sys
+from wxflow import AttrDict, parse_j2yaml
+from argparse import ArgumentParser
+from pathlib import Path
+
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..'))
+
+description = """parse yaml file and return value of key"""
+
+
+def parse_args():
+    """
+    Parse command-line arguments.
+
+    Returns:
+        argparse.Namespace: The parsed command-line arguments.
+    """
+
+    parser = ArgumentParser(description=description)
+    parser.add_argument('-y', '--yaml', help='full path to yaml file to parse', type=Path, required=True)
+    parser.add_argument('-k', '--key', help='key to return value of', type=str, required=True)
+    parser.add_argument('-s', '--string', help='output results as strings', action="store_true", required=False)
+    return parser.parse_args()
+
+
+def yq(yamlfile, key):
+    """
+    Parse a yaml file and return the value of a specified key.
+
+    Args:
+        yamlfile (Path): The path to the yaml file.
+        key (str): The key to return the value of.
+
+    Returns:
+        The value of the specified key in the yaml file.
+    """
+
+    data = AttrDict(HOMEgfs=_top)
+    data.update({'HOMEgfs': _top})
+    ydict = parse_j2yaml(path=yamlfile, data=data)
+    if key == 'all':
+        return ydict
+    list_keys = key.split('.')
+    for k in list_keys:
+        ydict = ydict.get(k, None)
+        if ydict is None:
+            break
+    return ydict
+
+
+if __name__ == '__main__':
+    """
+    Main function. Parses command-line arguments and prints the value of the specified key in the specified yaml file.
+    """
+
+    args = parse_args()
+    values = yq(args.yaml, args.key)
+    if args.string and isinstance(values, list):
+        for value in values:
+            print(value)
+    else:
+        print(values)