Skip to content

Commit

Permalink
Fix parallel longitudinal processing
Browse files Browse the repository at this point in the history
- Add --debug flag to long_fastsurfer to get debug output for brun calls
- Fix Shellcheck warnings
- Remove temporary manual loops
- Add positional arguments for segmentation in long_prepare
  • Loading branch information
dkuegler authored and m-reuter committed Nov 20, 2024
1 parent bb51a86 commit 786fa4d
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 74 deletions.
100 changes: 40 additions & 60 deletions long_fastsurfer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ tpids=()
t1s=()
parallel=0
log=""
brun_flags=()
python="python3.10 -s" # avoid user-directory package inclusion


Expand Down Expand Up @@ -174,6 +175,7 @@ case $key in
echo " pipeline run is a valid longitudinal run!"
exit 1
;;
--debug) brun_flags+=("$key") ;;
*) # unknown option
POSITIONAL_FASTSURFER[i]=$key
i=$((i + 1))
Expand Down Expand Up @@ -235,29 +237,32 @@ if [[ -z "$LF" ]]
then
LF="$sd/$tid/scripts/long_fastsurfer.log"
fi
function log () { echo "$1" | tee -a "$LF" ; }


################################### Prepare Base ##################################

echo "Base Setup $tid"
log "Base Setup $tid"
cmda=("$reconsurfdir/long_prepare_template.sh"
--tid "$tid" --t1s "${t1s[@]}" --tpids "${tpids[@]}"
--py "$python"
"${POSITIONAL_FASTSURFER[@]}")
# run_it will exit the bash script if the command fails (with exit code 1)
run_it "$LF" "${cmda[@]}"

################################### Run Base Seg ##################################

echo "Base Seg $tid"
log "Base Segmentation $tid"
cmda=("$FASTSURFER_HOME/run_fastsurfer.sh"
--sid "$tid" --sd "$sd" --base
--seg_only --py "$python"
"${POSITIONAL_FASTSURFER[@]}")
# run_it will exit the bash script if the command fails (with exit code 1)
run_it "$LF" "${cmda[@]}"

################################### Run Base Surf #################################

echo "Base Surf $tid"
log "Base Surface reconstruction $tid"
cmda=("$FASTSURFER_HOME/run_fastsurfer.sh"
--sid "$tid" --sd "$sd"
--surf_only --base --py "$python"
Expand All @@ -271,33 +276,23 @@ if [[ "$parallel" == "1" ]] ; then
} > "$base_surf_cmdf_log"
echo "#/bin/bash" > "$base_surf_cmdf"
run_it_cmdf "$LF" "$base_surf_cmdf" "${cmda[@]}"
bash "$base_surf_cmdf" 2>&1 >> "$base_surf_cmdf_log" &
bash "$base_surf_cmdf" >> "$base_surf_cmdf_log" 2>&1 &
base_surf_pid=$!
# shellcheck disable=SC2064
trap "if [[ -n \"\$(ps --no-headers $base_surf_pid)\" ]] ; then kill $base_surf_pid ; fi" EXIT
else
run_it "$LF" "${cmda[@]}"
fi

################################### Run Long Seg ##################################

# This can run in parallel with base seg and surf steps above
for ((i=0;i<${#tpids[@]};++i)); do
echo "Long Seg: ${tpids[i]} with T1 ${t1s[i]}"
cmd="$FASTSURFER_HOME/run_fastsurfer.sh \
--sid ${tpids[i]} --sd $sd \
--seg_only --long $tid \
${POSITIONAL_FASTSURFER[*]}"
RunIt "$cmd" "$LF"
done

# skip this for now as brun does not even have the --long flag yet
if false ; then
time_points=()
for ((i=0;i<${#tpids[@]};++i)); do
time_points+=("${tpids[$i]}=from-base")
done
cmda=("$FASTSURFER_HOME/brun_fastsurfer.sh" --subjects "${time_points[@]}" --sd "$sd" --seg_only --long "$tid"
"${POSITIONAL_FASTSURFER[@]}")
"${brun_flags[@]}" "${POSITIONAL_FASTSURFER[@]}")

if [[ "$parallel" == "1" ]] ; then
long_seg_cmdf="$SUBJECTS_DIR/$tid/scripts/long_seg.cmdf"
Expand All @@ -312,68 +307,53 @@ if [[ "$parallel" == "1" ]] ; then
# Surfaces in different jobs. Alternative, add a command to "$long_seg_cmdf" that releases the gpu or
# triggers the next "subject"
#TQDM_DISABLE=1
bash "$long_seg_cmdf" 2>&1 >> "$long_seg_cmdf_log" &
bash "$long_seg_cmdf" >> "$long_seg_cmdf_log" 2>&1 &
long_seg_pid=$!
# shellcheck disable=SC2064
trap "if [[ -n \"\$(ps --no-headers $long_seg_pid)\" ]] ; then kill $long_seg_pid ; fi" EXIT
else
run_it "$LF" "${cmda[@]}"
fi
fi # comment block

################################### Run Long Surf #################################

for ((i=0;i<${#tpids[@]};++i)); do
echo "Long Surf: ${tpids[i]} with T1 ${t1s[i]}"
cmd="$FASTSURFER_HOME/run_fastsurfer.sh \
--sid ${tpids[i]} --sd $sd \
--surf_only --long $tid \
${POSITIONAL_FASTSURFER[*]}"
RunIt "$cmd" "$LF"
done

# skip this for now as brun does not even have the --long flag yet
if false ; then
cmda=("$FASTSURFER_HOME/brun_fastsurfer.sh" --subjects "${time_points[@]}" --sd "$sd" --surf_only --long "$tid"
"${POSITIONAL_FASTSURFER[@]}")
"${brun_flags[@]}" "${POSITIONAL_FASTSURFER[@]}")
if [[ "$parallel" == "1" ]] ; then
cmda+=("--parallel_subjects")

# Append the base surface and longitudinal segmentation logs, exit if either failed
what_failed=()
wait $base_surf_pid
wait "$base_surf_pid"
success1=$?
{
echo "Base Surface pipeline Log:"
echo "======================================="
cat "$base_surf_cmdf_log"
if [ "$success1" -ne 0 ] ; then
echo "Base Surface pipeline terminated with error: $success1"
what_failed+=("Base Surface Pipeline")
else
echo "Base Surface pipeline finished successful!"
rm "$base_surf_cmdf_log" # the content of this file is transferred to LF
fi
echo "======================================="
} | tee -a "$LF"
wait $long_seg_pid
log "Base Surface pipeline Log:"
log "======================================="
tee -a "$LF" < "$base_surf_cmdf_log"
if [ "$success1" -ne 0 ] ; then
log "Base Surface pipeline terminated with error: $success1"
what_failed+=("Base Surface Pipeline")
else
log "Base Surface pipeline finished successful!"
rm "$base_surf_cmdf_log" # the content of this file is transferred to LF
fi
log "======================================="
wait "$long_seg_pid"
success2=$?
{
echo "Longitudinal Segmentation pipeline Log:"
echo "======================================="
cat "$long_seg_cmdf_log"
if [ "$success2" -ne 0 ] ; then
echo "Longitudinal Segmentation pipeline terminated with error: $success2"
what_failed+=("Longitudinal Segmentation Pipeline")
else
echo "Longitudinal Segmentation pipeline finished successful!"
rm "$long_seg_cmdf_log" # the content of this file is transferred to LF
fi
echo "======================================="
log "Longitudinal Segmentation pipeline Log:"
log "======================================="
tee -a "$LF" < "$long_seg_cmdf_log"
if [ "$success2" -ne 0 ] ; then
log "Longitudinal Segmentation pipeline terminated with error: $success2"
what_failed+=("Longitudinal Segmentation Pipeline")
else
log "Longitudinal Segmentation pipeline finished successful!"
rm "$long_seg_cmdf_log" # the content of this file is transferred to LF
fi
log "======================================="
if [[ "$success1" -ne 0 ]] || [[ "$success2" -ne 0 ]] ; then
echo "Terminating because ${what_failed[*]} failed!"
log "Terminating because ${what_failed[*]} failed!"
exit 1
fi
} | tee -a "$LF"
fi
run_it "$LF" "${cmda[@]}"
fi # comment block

33 changes: 19 additions & 14 deletions recon_surf/long_prepare_template.sh
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ i=0
while [[ $# -gt 0 ]]
do
# make key lowercase
key=$(echo "$1" | tr '[:upper:]' '[:lower:]')
arg="$1"
key=$(echo "$arg" | tr '[:upper:]' '[:lower:]')

shift # past argument
case $key in
Expand Down Expand Up @@ -194,8 +195,8 @@ case $key in
esac
shift # past value
;;
*) # unknown option get ignored
POSITIONAL_FASTSURFER[i]=$key
*) # unknown options get forwarded to run_fastsurfer (or ignored)
POSITIONAL_FASTSURFER[i]="$arg"
i=$((i + 1))
;;
#*) # unknown option
Expand Down Expand Up @@ -273,14 +274,17 @@ fi
VERSION=$($python "$FASTSURFER_HOME/FastSurferCNN/version.py" "${version_args[@]}")
echo "Version: $VERSION" | tee -a "$LF"
echo "Log file for long_prepare_template" >> "$LF"
{ date 2>&1 ; echo "" ; } | tee -a "$LF"
echo "" | tee -a "$LF"
echo "export SUBJECTS_DIR=$SUBJECTS_DIR" | tee -a "$LF"
echo "cd `pwd`" | tee -a "$LF"
echo "$0 ${inputargs[*]}" | tee -a $LF
echo "" | tee -a "$LF"
cat "$FREESURFER_HOME/build-stamp.txt" 2>&1 | tee -a "$LF"
uname -a 2>&1 | tee -a "$LF"
{
date 2>&1
echo ""
echo ""
echo "export SUBJECTS_DIR=$SUBJECTS_DIR"
echo "cd `pwd`"
echo "$0 ${inputargs[*]}"
echo ""
cat "$FREESURFER_HOME/build-stamp.txt" 2>&1
uname -a 2>&1
} | tee -a "$LF"


### IF THE SCRIPT GETS TERMINATED, ADD A MESSAGE
Expand Down Expand Up @@ -343,11 +347,11 @@ for ((i=0;i<${#tpids[@]};++i)); do
#printf "%s with T1 %s\n" "${tpids[i]}" "${t1s[i]}"
echo "${tpids[i]} with T1 ${t1s[i]}" | tee -a "$LF"
mdir="$SUBJECTS_DIR/$tid/long-inputs/${tpids[i]}"
mkdir -p $mdir
mkdir -p "$mdir"
# Import (copy) raw inputs (convert to extension format)
t1input=$mdir/cross_input${extension}
cmd="mri_convert ${t1s[i]} $t1input"
RunIt "$cmd" $LF
RunIt "$cmd" "$LF"

# conform !!!!!!! should we conform to some common value, determined from all time points?? !!!!!!
# this is relevant if input resolutions differe (which they should not), currently conform min may not work as expected
Expand All @@ -364,7 +368,8 @@ for ((i=0;i<${#tpids[@]};++i)); do
mask_name="$mdir/cross_mask${extension}"
aseg_segfile="$mdir/cross_aseg.auto_noCCseg${extension}"
seg_log="/dev/null"
cmd=($python "$fastsurfercnndir/run_prediction.py" --t1 "$t1input"
cmd=($python "$fastsurfercnndir/run_prediction.py"
"${POSITIONAL_FASTSURFER[@]}" --t1 "$t1input"
--asegdkt_segfile "$asegdkt_segfile" --conformed_name "$conformed_name"
--brainmask_name "$mask_name" --aseg_name "$aseg_segfile" --sid "${tpids[i]}"
--seg_log "$seg_log" --vox_size "$vox_size" --batch_size "$batch_size"
Expand Down

0 comments on commit 786fa4d

Please sign in to comment.