From d87a550fb2a635221744db453d6d880e92523b70 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 3 Jun 2024 08:27:06 -0500 Subject: [PATCH 01/13] Optimize the wavepostpnt scripts --- scripts/exgfs_wave_post_pnt.sh | 14 ++++---------- ush/wave_outp_spec.sh | 16 ++++++++-------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 6e456e2aec..7f04140e68 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -156,7 +156,7 @@ source "${USHgfs}/preamble.sh" cp -f ${PARMgfs}/wave/wave_${NET}.buoys buoy.loc.temp if [ "$DOBNDPNT_WAV" = YES ]; then #only do boundary points - sed -n '/^\$.*/!p' buoy.loc.temp | grep IBP > buoy.loc || { + sed -n '/^\$.*IBP.*/!p' buoy.loc.temp > buoy.loc || { echo "WARNING: No boundary points found in buoy file ${PARMgfs}/wave/wave_${NET}.buoys" echo " Ending job without doing anything." exit 0 @@ -289,7 +289,7 @@ source "${USHgfs}/preamble.sh" fi # Create new buoy_log.ww3 - cat buoy.loc | awk '{print $3}' | sed 's/'\''//g' > ibp_tags + awk '{print $3}' buoy.loc | sed 's/'\''//g' > ibp_tags grep -F -f ibp_tags buoy_log.ww3 > buoy_log.tmp rm -f buoy_log.dat mv buoy_log.tmp buoy_log.dat @@ -385,19 +385,13 @@ source "${USHgfs}/preamble.sh" if [ "$DOSPC_WAV" = 'YES' ] then export dtspec=3600. - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_spec.sh $buoy $ymdh spec $SPECDATA > $SPECDATA/spec_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - done + echo ${bouys} | sed "g/(^.*$)/${USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${SPECDATA} > ${SPECDATA}\/spec_\1.out 2>&1/" > tmpcmdfile.${FH3} fi if [ "$DOBLL_WAV" = 'YES' ] then export dtspec=3600. - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - done + echo ${bouys} | sed "g/(^.*$)/${USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${SPECDATA} > ${SPECDATA}\/bull_\1.out 2>&1/" > tmpcmdfile.${FH3} fi split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh index db9997fd54..4a95463e6b 100755 --- a/ush/wave_outp_spec.sh +++ b/ush/wave_outp_spec.sh @@ -200,8 +200,8 @@ source "${USHgfs}/preamble.sh" then if [ "$specdir" = "bull" ] then - cat $outfile | sed -e '9,$d' >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - cat $coutfile | sed -e '8,$d' >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '9,$d' "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull + sed '8,$d' "${coutfile}" >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull else cat $outfile >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec fi @@ -209,18 +209,18 @@ source "${USHgfs}/preamble.sh" then if [ "$specdir" = "bull" ] then - cat $outfile | sed -e '1,7d' >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - cat $coutfile | sed -e '1,6d' >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '1,7d' "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull + sed '1,6d' "${coutfile}" >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull else - cat $outfile | sed -n "/^${YMD} ${HMS}$/,\$p" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec fi else if [ "$specdir" = "bull" ] then - cat $outfile | sed -e '1,7d' | sed -e '2,$d' >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - cat $coutfile | sed -e '1,6d' | sed -e '2,$d' >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '8q;d' "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull + sed '7q;d' "${coutfile}" >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull else - cat $outfile | sed -n "/^${YMD} ${HMS}$/,\$p" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec fi fi else From 709e6dc0a4d8f35716b5b6bd930d3b0c9e19a244 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 4 Jun 2024 12:27:43 -0500 Subject: [PATCH 02/13] Further optimize the wavepostpnt job --- scripts/exgfs_wave_post_pnt.sh | 42 +++++++++++++++------------------- ush/wave_outp_spec.sh | 21 +++++------------ 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 7f04140e68..9eee4cd71d 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -296,7 +296,7 @@ source "${USHgfs}/preamble.sh" grep -F -f ibp_tags buoy_lst.loc > buoy_tmp1.loc #sed '$d' buoy_tmp1.loc > buoy_tmp2.loc - buoys=$(awk '{ print $1 }' buoy_tmp1.loc) + awk '{ print $1 }' buoy_tmp1.loc > buoy_lst.txt Nb=$(wc buoy_tmp1.loc | awk '{ print $1 }') rm -f buoy_tmp1.loc @@ -385,13 +385,13 @@ source "${USHgfs}/preamble.sh" if [ "$DOSPC_WAV" = 'YES' ] then export dtspec=3600. - echo ${bouys} | sed "g/(^.*$)/${USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${SPECDATA} > ${SPECDATA}\/spec_\1.out 2>&1/" > tmpcmdfile.${FH3} + sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} spec ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/spec_\1.out 2>&1/" buoy_lst.txt >> tmpcmdfile.${FH3} fi if [ "$DOBLL_WAV" = 'YES' ] then export dtspec=3600. - echo ${bouys} | sed "g/(^.*$)/${USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${SPECDATA} > ${SPECDATA}\/bull_\1.out 2>&1/" > tmpcmdfile.${FH3} + sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} bull ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/bull_\1.out 2>&1/" buoy_lst.txt >> tmpcmdfile.${FH3} fi split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 @@ -497,27 +497,21 @@ source "${USHgfs}/preamble.sh" cd $DATA - echo "Before create cmdfile for cat bouy : $(date)" - rm -f cmdfile.bouy - touch cmdfile.bouy - chmod 744 cmdfile.bouy + echo "Before create cmdfile for cat buoy : $(date)" + rm -f cmdfile.buoy + touch cmdfile.buoy + chmod 744 cmdfile.buoy CATOUTDIR=${DATA}/pnt_cat_out mkdir -p ${CATOUTDIR} if [ "$DOSPC_WAV" = 'YES' ] then - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT spec > ${CATOUTDIR}/spec_cat_$buoy.out 2>&1" >> cmdfile.bouy - done + sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${CATOUTDIR//\//\\\/}\/spec_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy fi if [ "$DOBLL_WAV" = 'YES' ] then - for buoy in $buoys - do - echo "${USHgfs}/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy - done + sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${CATOUTDIR//\//\\\/}\/bull_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy fi if [ ${CFP_MP:-"NO"} = "YES" ]; then @@ -525,18 +519,18 @@ source "${USHgfs}/preamble.sh" ifile=0 iline=1 ifirst='yes' - nlines=$( wc -l cmdfile.bouy | awk '{print $1}' ) + nlines=$( wc -l cmdfile.buoy | awk '{print $1}' ) while [ $iline -le $nlines ]; do - line=$( sed -n ''$iline'p' cmdfile.bouy ) + line=$( sed -n ''$iline'p' cmdfile.buoy ) if [ -z "$line" ]; then break else if [ "$ifirst" = 'yes' ]; then - echo "#!/bin/sh" > cmdfile.bouy.$nfile - echo "$nfile cmdfile.bouy.$nfile" >> cmdmprogbouy - chmod 744 cmdfile.bouy.$nfile + echo "#!/bin/sh" > cmdfile.buoy.$nfile + echo "$nfile cmdfile.buoy.$nfile" >> cmdmprogbuoy + chmod 744 cmdfile.buoy.$nfile fi - echo $line >> cmdfile.bouy.$nfile + echo $line >> cmdfile.buoy.$nfile nfile=$(( nfile + 1 )) if [ $nfile -eq $NTASKS ]; then nfile=0 @@ -547,7 +541,7 @@ source "${USHgfs}/preamble.sh" done fi - wavenproc=$(wc -l cmdfile.bouy | awk '{print $1}') + wavenproc=$(wc -l cmdfile.buoy | awk '{print $1}') wavenproc=$(echo $((${wavenproc}<${NTASKS}?${wavenproc}:${NTASKS}))) set +x @@ -560,9 +554,9 @@ source "${USHgfs}/preamble.sh" if [ "$wavenproc" -gt '1' ] then if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdmprogbouy + ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdmprogbuoy else - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile.bouy + ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile.buoy fi exit=$? else diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh index 4a95463e6b..fa5bebc401 100755 --- a/ush/wave_outp_spec.sh +++ b/ush/wave_outp_spec.sh @@ -73,21 +73,7 @@ source "${USHgfs}/preamble.sh" exit 1 else buoy=$bloc - grep $buoy ${DATA}/buoy_log.ww3 > tmp_list.loc - while read line - do - buoy_name=$(echo $line | awk '{print $2}') - if [ $buoy = $buoy_name ] - then - point=$(echo $line | awk '{ print $1 }') - set +x - echo " Location ID/# : $buoy (${point})" - echo " Spectral output start time : $ymdh " - echo ' ' - set_trace - break - fi - done < tmp_list.loc + point=$(awk "{if (\$2 == \"${buoy}\"){print \$1; exit} }" "${DATA}/buoy_log.ww3") if [ -z "$point" ] then set +x @@ -97,6 +83,11 @@ source "${USHgfs}/preamble.sh" echo ' ' set_trace exit 2 + else + set +x + echo " Location ID/# : $buoy (${point})" + echo " Spectral output start time : $ymdh " + echo ' ' fi fi From 0acaa37f3d4a21f7ab8a3101aad2a09580347f8f Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 4 Jun 2024 12:51:39 -0500 Subject: [PATCH 03/13] Add double quotes around variables --- scripts/exgfs_wave_post_pnt.sh | 8 ++++---- ush/wave_outp_spec.sh | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 9eee4cd71d..b03881cbeb 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -385,13 +385,13 @@ source "${USHgfs}/preamble.sh" if [ "$DOSPC_WAV" = 'YES' ] then export dtspec=3600. - sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} spec ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/spec_\1.out 2>&1/" buoy_lst.txt >> tmpcmdfile.${FH3} + sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} spec ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/spec_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi if [ "$DOBLL_WAV" = 'YES' ] then export dtspec=3600. - sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} bull ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/bull_\1.out 2>&1/" buoy_lst.txt >> tmpcmdfile.${FH3} + sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} bull ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/bull_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 @@ -554,9 +554,9 @@ source "${USHgfs}/preamble.sh" if [ "$wavenproc" -gt '1' ] then if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdmprogbuoy + ${wavempexec} -n "${wavenproc}" "${wave_mpmd}" cmdmprogbuoy else - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile.buoy + ${wavempexec} "${wavenproc}" "${wave_mpmd}" cmdfile.buoy fi exit=$? else diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh index fa5bebc401..3cf7decef4 100755 --- a/ush/wave_outp_spec.sh +++ b/ush/wave_outp_spec.sh @@ -191,27 +191,27 @@ source "${USHgfs}/preamble.sh" then if [ "$specdir" = "bull" ] then - sed '9,$d' "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - sed '8,$d' "${coutfile}" >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '9,$d' "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.bull" + sed '8,$d' "${coutfile}" >> "${STA_DIR}/c${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.cbull" else - cat $outfile >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + cat $outfile >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.spec" fi elif [ "${ymdh}" = "${YMDHE}" ] then if [ "$specdir" = "bull" ] then - sed '1,7d' "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - sed '1,6d' "${coutfile}" >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '1,7d' "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.bull" + sed '1,6d' "${coutfile}" >> "${STA_DIR}/c${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.cbull" else - sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.spec" fi else if [ "$specdir" = "bull" ] then - sed '8q;d' "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.bull - sed '7q;d' "${coutfile}" >> ${STA_DIR}/c${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.cbull + sed '8q;d' "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.bull" + sed '7q;d' "${coutfile}" >> "${STA_DIR}/c${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.cbull" else - sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> ${STA_DIR}/${specdir}fhr/$WAV_MOD_TAG.${ymdh}.$buoy.spec + sed -n "/^${YMD} ${HMS}$/,\$p" "${outfile}" >> "${STA_DIR}/${specdir}fhr/${WAV_MOD_TAG}.${ymdh}.${buoy}.spec" fi fi else From db300ebfb8a22493021cd62fe5d5453ad54b8cbc Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 4 Jun 2024 19:01:18 +0000 Subject: [PATCH 04/13] Apply suggestions from code review Remove unnecessary awk calls. Co-authored-by: Walter Kolczynski - NOAA --- scripts/exgfs_wave_post_pnt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index b03881cbeb..54e3056f8a 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -519,7 +519,7 @@ source "${USHgfs}/preamble.sh" ifile=0 iline=1 ifirst='yes' - nlines=$( wc -l cmdfile.buoy | awk '{print $1}' ) + nlines=$( wc -l < cmdfile.buoy) while [ $iline -le $nlines ]; do line=$( sed -n ''$iline'p' cmdfile.buoy ) if [ -z "$line" ]; then @@ -541,7 +541,7 @@ source "${USHgfs}/preamble.sh" done fi - wavenproc=$(wc -l cmdfile.buoy | awk '{print $1}') + wavenproc=$(wc -l < cmdfile.buoy) wavenproc=$(echo $((${wavenproc}<${NTASKS}?${wavenproc}:${NTASKS}))) set +x From 8f4f7ab6378975a837c1beba44d802e8273a4863 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 5 Jun 2024 06:26:05 -0500 Subject: [PATCH 05/13] Pregenerate sed-searchable paths. --- scripts/exgfs_wave_post_pnt.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 54e3056f8a..1e411acf66 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -349,6 +349,8 @@ source "${USHgfs}/preamble.sh" # 1.a.2 Loop over forecast time to generate post files fhr=$FHMIN_WAV + # Generated sed-searchable paths + escaped_USHgfs="${USHgfs//\//\\\/}" while [ $fhr -le $FHMAX_WAV_PNT ]; do echo " Creating the wave point scripts at : $(date)" @@ -365,6 +367,7 @@ source "${USHgfs}/preamble.sh" # Create instances of directories for spec and gridded output export SPECDATA=${DATA}/output_$YMDHMS + escaped_SPECDATA="${SPECDATA//\//\\\/}" export BULLDATA=${DATA}/output_$YMDHMS cp $DATA/mod_def.${waveuoutpGRD} mod_def.${waveuoutpGRD} @@ -385,13 +388,15 @@ source "${USHgfs}/preamble.sh" if [ "$DOSPC_WAV" = 'YES' ] then export dtspec=3600. - sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} spec ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/spec_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" + # Construct the wave_outp_spec (spec) command to run on each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi if [ "$DOBLL_WAV" = 'YES' ] then export dtspec=3600. - sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_spec.sh \1 ${ymdh} bull ${SPECDATA//\//\\\/} > ${SPECDATA//\//\\\/}\/bull_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" + # Construct the wave_outp_spec (bull) command to run on each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 @@ -502,16 +507,19 @@ source "${USHgfs}/preamble.sh" touch cmdfile.buoy chmod 744 cmdfile.buoy CATOUTDIR=${DATA}/pnt_cat_out + escaped_CATOUTDIR="${CATOUTDIR//\//\\\/}" mkdir -p ${CATOUTDIR} if [ "$DOSPC_WAV" = 'YES' ] then - sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${CATOUTDIR//\//\\\/}\/spec_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy + # Construct wave_outp_cat (spec) call for each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${escaped_CATOUTDIR}\/spec_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy fi if [ "$DOBLL_WAV" = 'YES' ] then - sed "s/^\(.*\)$/${USHgfs//\//\\\/}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${CATOUTDIR//\//\\\/}\/bull_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy + # Construct wave_outp_cat (bull) call for each buoy in buoy_lst.txt + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${escaped_CATOUTDIR}\/bull_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy fi if [ ${CFP_MP:-"NO"} = "YES" ]; then From 670a7ef58c08733bec77cd82f01406da43350e1e Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 7 Jun 2024 08:18:04 -0500 Subject: [PATCH 06/13] Revert buggy optimization for finding IBP points --- scripts/exgfs_wave_post_pnt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 1e411acf66..d4733aa809 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -156,7 +156,7 @@ source "${USHgfs}/preamble.sh" cp -f ${PARMgfs}/wave/wave_${NET}.buoys buoy.loc.temp if [ "$DOBNDPNT_WAV" = YES ]; then #only do boundary points - sed -n '/^\$.*IBP.*/!p' buoy.loc.temp > buoy.loc || { + sed -n '/^\$.*/!p' buoy.loc.temp | grep IBP > buoy.loc || { echo "WARNING: No boundary points found in buoy file ${PARMgfs}/wave/wave_${NET}.buoys" echo " Ending job without doing anything." exit 0 From 2d94656c640dd2673711af26cbc522d30a6995e0 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 7 Jun 2024 09:30:24 -0500 Subject: [PATCH 07/13] Escape '&' in sed commands --- scripts/exgfs_wave_post_pnt.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index d4733aa809..eff162cd26 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -389,14 +389,14 @@ source "${USHgfs}/preamble.sh" then export dtspec=3600. # Construct the wave_outp_spec (spec) command to run on each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi if [ "$DOBLL_WAV" = 'YES' ] then export dtspec=3600. # Construct the wave_outp_spec (bull) command to run on each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" fi split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 @@ -513,13 +513,13 @@ source "${USHgfs}/preamble.sh" if [ "$DOSPC_WAV" = 'YES' ] then # Construct wave_outp_cat (spec) call for each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${escaped_CATOUTDIR}\/spec_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${escaped_CATOUTDIR}\/spec_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy fi if [ "$DOBLL_WAV" = 'YES' ] then # Construct wave_outp_cat (bull) call for each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${escaped_CATOUTDIR}\/bull_cat_\1.out 2>&1/" buoy_lst.txt >> cmdfile.buoy + sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${escaped_CATOUTDIR}\/bull_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy fi if [ ${CFP_MP:-"NO"} = "YES" ]; then From 0b4a6da72731af9a6f9125f109dd1dc40de4a1ca Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 7 Jun 2024 10:58:46 -0500 Subject: [PATCH 08/13] Remove double-quotes from wave_mpmd --- scripts/exgfs_wave_post_pnt.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index eff162cd26..4236426f8d 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -562,9 +562,11 @@ source "${USHgfs}/preamble.sh" if [ "$wavenproc" -gt '1' ] then if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n "${wavenproc}" "${wave_mpmd}" cmdmprogbuoy + # shellcheck disable=SC2086 + ${wavempexec} -n "${wavenproc}" ${wave_mpmd} cmdmprogbuoy else - ${wavempexec} "${wavenproc}" "${wave_mpmd}" cmdfile.buoy + # shellcheck disable=SC2086 + ${wavempexec} "${wavenproc}" ${wave_mpmd} cmdfile.buoy fi exit=$? else From 1e49ab95bbee3d22181bb7073af559e8336e4f15 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 10 Jun 2024 08:06:07 -0500 Subject: [PATCH 09/13] Increase wallclock for wavepostbndpnt jobs --- parm/config/gefs/config.resources | 2 +- parm/config/gfs/config.resources | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 04d55ae082..b2ca5b3e51 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -255,7 +255,7 @@ case ${step} in # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. # Limit the number of tasks/node to 40. "wavepostbndpnt") - export wtime_wavepostbndpnt="01:00:00" + export wtime_wavepostbndpnt="03:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt )) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 0972f74f9c..d9d964a69b 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -144,7 +144,7 @@ case ${step} in # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. # Limit the number of tasks/node to 40. "wavepostbndpnt") - export wtime_wavepostbndpnt="01:00:00" + export wtime_wavepostbndpnt="03:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt )) From a7522cff68fc975b52e799560a7232a7638b44b0 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 10 Jun 2024 08:07:56 -0500 Subject: [PATCH 10/13] Adjust data requirement for wavepostbndpntbll job for experiment forecast hours --- workflow/rocoto/gefs_tasks.py | 9 ++++++++- workflow/rocoto/gfs_tasks.py | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 86be494549..aa42573591 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -387,7 +387,14 @@ def wavepostbndpntbll(self): deps = [] atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) # Is there any reason this is 180? - data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf180.txt' + # The wavepostbndpntbll job runs on forecast hours up to 180 hours + last_fhr = Tasks._get_forecast_hours(self.cdump, + self._configs['wavepostbndpntbll'], + component='atmos')[-1] + if last_fhr > 180: + last_fhr = 180 + + data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf{last_fhr:03d}.txt' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index fa218c6713..eef1b137ae 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1175,9 +1175,17 @@ def wavepostbndpnt(self): return task def wavepostbndpntbll(self): + + # The wavepostbndpntbll job runs on forecast hours up to 180 hours + last_fhr = Tasks._get_forecast_hours(self.cdump, + self._configs['wavepostbndpntbll'], + component='atmos')[-1] + if last_fhr > 180: + last_fhr = 180 + deps = [] atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"]) - data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf180.txt' + data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf{last_fhr:03d}.txt' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) From 6dfedcadba6b82552b6f84fd3c40568547e07554 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 10 Jun 2024 14:45:55 -0500 Subject: [PATCH 11/13] Key wavepostbndpntbll log file off of FHMAX_WAV_IBP --- workflow/rocoto/gefs_tasks.py | 13 ++++++++----- workflow/rocoto/gfs_tasks.py | 11 +++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index aa42573591..6c2dca1955 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -386,13 +386,16 @@ def wavepostbndpnt(self): def wavepostbndpntbll(self): deps = [] atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) - # Is there any reason this is 180? - # The wavepostbndpntbll job runs on forecast hours up to 180 hours + + # The wavepostbndpntbll job runs on forecast hours up to FHMAX_WAV_IBP + wave_cfg = self._configs['wave'] last_fhr = Tasks._get_forecast_hours(self.cdump, - self._configs['wavepostbndpntbll'], + wave_cfg, component='atmos')[-1] - if last_fhr > 180: - last_fhr = 180 + + fhmax_wav_ibp = wave_cfg['FHMAX_WAV_IBP'] + if last_fhr > fhmax_wav_ibp: + last_fhr = fhmax_wav_ibp data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf{last_fhr:03d}.txt' dep_dict = {'type': 'data', 'data': data} diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index eef1b137ae..1c18b85a93 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1176,12 +1176,15 @@ def wavepostbndpnt(self): def wavepostbndpntbll(self): - # The wavepostbndpntbll job runs on forecast hours up to 180 hours + # The wavepostbndpntbll job runs on forecast hours up to FHMAX_WAV_IBP + wave_cfg = self._configs['wave'] last_fhr = Tasks._get_forecast_hours(self.cdump, - self._configs['wavepostbndpntbll'], + wave_cfg, component='atmos')[-1] - if last_fhr > 180: - last_fhr = 180 + + fhmax_wav_ibp = wave_cfg['FHMAX_WAV_IBP'] + if last_fhr > fhmax_wav_ibp: + last_fhr = fhmax_wav_ibp deps = [] atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"]) From 9817f1664e38275cfef908500819f0d3937cb892 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 11 Jun 2024 07:10:57 -0500 Subject: [PATCH 12/13] Correct FHMAX_WAV_IBP cap hours --- parm/config/gefs/config.wave | 2 +- parm/config/gfs/config.wave | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.wave b/parm/config/gefs/config.wave index b61a2f6e54..bef3437adf 100644 --- a/parm/config/gefs/config.wave +++ b/parm/config/gefs/config.wave @@ -87,7 +87,7 @@ export WAVEWND_FID= # The start time reflects the number of hindcast hours prior to the cycle initial time export WAVHINDH=0 export FHMAX_WAV_IBP=180 -if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_GFS} ; fi +if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_WAV} ; fi # gridded and point output rate export DTFLD_WAV=$(( FHOUT_HF_WAV * 3600 )) diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index 7253ef1396..568aeb1e1c 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -102,7 +102,7 @@ if [[ "${RUN}" == "gfs" ]]; then fi export WAVHINDH=0 export FHMAX_WAV_IBP=180 -if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_GFS} ; fi +if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_WAV} ; fi # gridded and point output rate export DTFLD_WAV=$(( FHOUT_HF_WAV * 3600 )) From 2e9953b486df8e084238bdd016801d5c8419e25a Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:24:15 +0000 Subject: [PATCH 13/13] Set last_fhr off of FHMAX_WAV_IBP Co-authored-by: Walter Kolczynski - NOAA --- workflow/rocoto/gefs_tasks.py | 9 +-------- workflow/rocoto/gfs_tasks.py | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 6c2dca1955..f112d29630 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -388,14 +388,7 @@ def wavepostbndpntbll(self): atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) # The wavepostbndpntbll job runs on forecast hours up to FHMAX_WAV_IBP - wave_cfg = self._configs['wave'] - last_fhr = Tasks._get_forecast_hours(self.cdump, - wave_cfg, - component='atmos')[-1] - - fhmax_wav_ibp = wave_cfg['FHMAX_WAV_IBP'] - if last_fhr > fhmax_wav_ibp: - last_fhr = fhmax_wav_ibp + last_fhr = self._configs['wave']['FHMAX_WAV_IBP'] data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf{last_fhr:03d}.txt' dep_dict = {'type': 'data', 'data': data} diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 1c18b85a93..d404ec7ada 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1177,14 +1177,7 @@ def wavepostbndpnt(self): def wavepostbndpntbll(self): # The wavepostbndpntbll job runs on forecast hours up to FHMAX_WAV_IBP - wave_cfg = self._configs['wave'] - last_fhr = Tasks._get_forecast_hours(self.cdump, - wave_cfg, - component='atmos')[-1] - - fhmax_wav_ibp = wave_cfg['FHMAX_WAV_IBP'] - if last_fhr > fhmax_wav_ibp: - last_fhr = fhmax_wav_ibp + last_fhr = self._configs['wave']['FHMAX_WAV_IBP'] deps = [] atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"])