Add hash to CI experiment paths to avoid collisions (#1735)

ci/cases/C48_S2S.yaml

@@ -3,10 +3,11 @@ experiment:
   mode: forecast-only
 
 arguments:
+  pslot: ${pslot}
   app: S2S
   resdet: 48
-  comrot: ${RUNTESTS}/${pslot}/COMROT
-  expdir: ${RUNTESTS}/${pslot}/EXPDIR
+  comrot: ${RUNTESTS}/COMROT
+  expdir: ${RUNTESTS}/EXPDIR
   idate: 2021032312
   edate: 2021032312
   yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml

ci/cases/C96C48_hybatmDA.yaml

@@ -3,11 +3,12 @@ experiment:
   mode: cycled
 
 arguments:
+  pslot: ${pslot}
   app: ATM
   resdet: 96
   resens: 48
-  comrot: ${RUNTESTS}/${pslot}/COMROT
-  expdir: ${RUNTESTS}/${pslot}/EXPDIR
+  comrot: ${RUNTESTS}/COMROT
+  expdir: ${RUNTESTS}/EXPDIR
   icsdir: ${ICSDIR_ROOT}/C96C48
   idate: 2021122018
   edate: 2021122106

ci/cases/C96_atm3DVar.yaml

@@ -3,10 +3,11 @@ experiment:
   mode: cycled
 
 arguments:
+  pslot: ${pslot}
   app: ATM
   resdet: 96
-  comrot: ${RUNTESTS}/${pslot}/COMROT
-  expdir: ${RUNTESTS}/${pslot}/EXPDIR
+  comrot: ${RUNTESTS}/COMROT
+  expdir: ${RUNTESTS}/EXPDIR
   icsdir: ${ICSDIR_ROOT}/C96C48
   idate: 2021122018
   edate: 2021122106

ci/scripts/check_ci.sh

@@ -6,7 +6,6 @@ set -eux
 #                     running rocotostat on each to determine if the experiment has
 #                     succeeded or faild.  This script is intended
 #                     to run from within a cron job in the CI Managers account
-# Abstract TODO
 #####################################################################################
 
 HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
@@ -78,11 +77,11 @@ for pr in ${pr_list}; do
   if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then
      continue
   fi
-  num_cases=$(find "${pr_dir}/RUNTESTS" -mindepth 1 -maxdepth 1 -type d | wc -l) || true
 
-  #Check for PR success  when ${pr_dir}/RUNTESTS is void of subfolders
+  #Check for PR success  when ${pr_dir}/RUNTESTS/EXPDIR is void of subfolders
   # since all successfull ones where previously removed
-  if [[ "${num_cases}" -eq 0 ]] && [[ -d "${pr_dir}/RUNTESTS" ]]; then
+  # shellcheck disable=SC2312
+  if [[ -z $(ls -A "${pr_dir}/RUNTESTS/EXPDIR") ]] ; then
     "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed"
     sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
@@ -92,14 +91,14 @@ for pr in ${pr_list}; do
     continue 
   fi
 
-  for cases in "${pr_dir}/RUNTESTS/"*; do
-    pslot=$(basename "${cases}") || true
+  for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do
+    pslot=$(basename "${pslot_dir}") || true
     if [[ -z "${pslot+x}" ]]; then
-      echo "No cases found in ${pr_dir}/RUNTESTS .. exiting"
+      echo "No experiments found in ${pslot_dir} .. exiting"
       exit 0
     fi
-    xml="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.xml"
-    db="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.db"
+    xml="${pslot_dir}/${pslot}.xml"
+    db="${pslot_dir}/${pslot}.db"
     if [[ ! -f "${db}" ]]; then
        continue
     fi
@@ -138,7 +137,8 @@ for pr in ${pr_list}; do
       sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
       #Remove Experment cases that completed successfully
-      rm -Rf "${pr_dir}/RUNTESTS/${pslot}"
+      rm -Rf "${pslot_dir}"
+      rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}"
     fi
   done
 done

ci/scripts/clone-build_ci.sh

@@ -78,7 +78,8 @@ echo "${commit}" > "../commit"
 # run checkout script
 cd sorc || exit 1
 set +e
-./checkout.sh -c -g -u >> log.checkout 2>&1
+# TODO enable -u later when GDASApp tests are added
+./checkout.sh -c -g >> log.checkout 2>&1
 checkout_status=$?
 if [[ ${checkout_status} != 0 ]]; then
   {

ci/scripts/create_experiment.py

@@ -20,6 +20,7 @@
 with an error code of 0 upon success.
 """
 
+import os
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
 
@@ -80,7 +81,6 @@ def input_args():
     setup_expt_args = YAMLFile(path=user_inputs.yaml)
 
     HOMEgfs = Path.absolute(Path(user_inputs.dir))
-    pslot = Path(user_inputs.yaml).stem
     type = setup_expt_args.experiment.type
     mode = setup_expt_args.experiment.mode
 
@@ -93,17 +93,14 @@ def input_args():
         setup_expt_cmd.add_default_arg(f'--{conf}')
         setup_expt_cmd.add_default_arg(str(value))
 
-    setup_expt_cmd.add_default_arg('--pslot')
-    setup_expt_cmd.add_default_arg(pslot)
-
     logger.info(f'Run command: {setup_expt_cmd.command}')
     setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr'))
     setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout'))
     print(setup_expt_stderr)
     setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr)
 
     setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py'))
-    expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(pslot)))
+    expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(setup_expt_args.arguments.pslot)))
     setup_xml_cmd.add_default_arg(str(expdir))
 
     logger.info(f'Run command: {setup_xml_cmd.command}')

ci/scripts/driver.sh

@@ -148,11 +148,15 @@ for pr in ${pr_list}; do
     #############################################################
     HOMEgfs_PR="${pr_dir}/global-workflow"
     export HOMEgfs_PR
+    cd "${HOMEgfs_PR}"
+    pr_sha=$(git rev-parse --short HEAD)
+
     for yaml_config in "${HOMEgfs_PR}/ci/cases/"*.yaml; do
-      pslot=$(basename "${yaml_config}" .yaml) || true
+      case=$(basename "${yaml_config}" .yaml) || true
+      pslot="${case}_${pr_sha}"
       export pslot
       set +e
-      "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${pslot}.yaml" --dir "${HOMEgfs_PR}"
+      "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" --dir "${HOMEgfs_PR}"
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then

ci/scripts/run_ci.sh

@@ -69,19 +69,19 @@ for pr in ${pr_list}; do
      continue
   fi
   num_cases=0
-  for cases in "${pr_dir}/RUNTESTS/"*; do
-    if [[ ! -d "${cases}" ]]; then
+  for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do
+    if [[ ! -d "${pslot_dir}" ]]; then
        continue
     fi
     ((num_cases=num_cases+1))
     # No more than two cases are going forward at a time for each PR
     if [[ "${num_cases}" -gt "${max_concurrent_cases}" ]]; then
        continue
     fi
-    pslot=$(basename "${cases}")
-    xml="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.xml"
-    db="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.db"
-    echo "Running: ${rocotorun} -v 6 -w ${xml} -d ${db}"
+    pslot=$(basename "${pslot_dir}")
+    xml="${pslot_dir}/${pslot}.xml"
+    db="${pslot_dir}/${pslot}.db"
+    echo "Running: ${rocotorun} -v 10 -w ${xml} -d ${db}"
     "${rocotorun}" -v 10 -w "${xml}" -d "${db}"
   done
 done