From 14f18a473b8afd586c6ed857a0afe43a903e9a24 Mon Sep 17 00:00:00 2001
From: Baxter Rogers <baxter.rogers@vanderbilt.edu>
Date: Thu, 25 Apr 2024 07:26:15 -0500
Subject: [PATCH 1/4] multijob

---
 bin/supplemental_tools/dax-testrun.sh | 31 ++++++++++++++++-----------
 1 file changed, 18 insertions(+), 13 deletions(-)
 mode change 100644 => 100755 bin/supplemental_tools/dax-testrun.sh

diff --git a/bin/supplemental_tools/dax-testrun.sh b/bin/supplemental_tools/dax-testrun.sh
old mode 100644
new mode 100755
index 578bde25..302eb98f
--- a/bin/supplemental_tools/dax-testrun.sh
+++ b/bin/supplemental_tools/dax-testrun.sh
@@ -85,7 +85,6 @@ dax build \
     --sessions "${session}" \
     "${setfile}"
 
-# FIXME Check for successful build or fail with useful info
 
 # Launch
 echo "Launching"
@@ -95,27 +94,33 @@ dax launch \
     --sessions "${session}" \
     "${setfile}"
 
-# Identify and track the job (check every $delay seconds)
-jobid=$(grep "INFO - cluster - Submitted batch job" "${launchlog}" | cut -d ' ' -f 11)
+# Identify and track the jobs (check every $delay seconds). Handle multiple
+# jobs launched
+jobidstr=$(grep "INFO - cluster - Submitted batch job" "${launchlog}" | cut -d ' ' -f 11)
+jobids=(${jobidstr//\n/ })
+jobcount=${#jobids[@]}
 
-if [ -z "${jobid}" ]; then
+if [ -z "${jobidstr}" ]; then
     echo "Job not launched"
     exit 1
 else
-    echo "Job ${jobid} launched"
+    echo "Jobs launched (${jobcount}): ${jobids[@]}"
 fi
 
-jobstate=
-while [ "${jobstate}" != "completed" ]; do
+jobscompleted=0
+while [ "${jobscompleted}" -lt "${jobcount}" ]; do
     sleep "${delay}"
-    jobstate=$(rtracejob ${jobid} |grep "State")
-    jobstate=$(echo ${jobstate##*|})
-    echo "Job ${jobid} state: ${jobstate}"
+    jobscompleted=0
+    for job in ${jobids[@]}; do
+        jobstate=$(rtracejob ${job} |grep "State")
+        jobstate=$(echo ${jobstate##*|})
+        echo "Job ${jobid} state: ${jobstate}"
+        if [ "${jobstate}" = "complete" ]; then
+            ((jobscompleted++))
+        fi
+    done
 done
 
-# FIXME show the assessor status (JOB_FAILED.txt, READY_TO_COMPLETE.txt, ...)
-
-# FIXME If failed, report error lines in outlog
 
 # Update/upload to get results to xnat
 echo Updating

From f03b64d3390928cb5c0f5628645821760db3afe5 Mon Sep 17 00:00:00 2001
From: Baxter Rogers <baxter.rogers@vanderbilt.edu>
Date: Thu, 25 Apr 2024 07:41:00 -0500
Subject: [PATCH 2/4] multiple-jobs

---
 bin/supplemental_tools/dax-testrun.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/supplemental_tools/dax-testrun.sh b/bin/supplemental_tools/dax-testrun.sh
index 302eb98f..0579b375 100755
--- a/bin/supplemental_tools/dax-testrun.sh
+++ b/bin/supplemental_tools/dax-testrun.sh
@@ -114,11 +114,12 @@ while [ "${jobscompleted}" -lt "${jobcount}" ]; do
     for job in ${jobids[@]}; do
         jobstate=$(rtracejob ${job} |grep "State")
         jobstate=$(echo ${jobstate##*|})
-        echo "Job ${jobid} state: ${jobstate}"
+        echo "Job ${job} state: ${jobstate}"
         if [ "${jobstate}" = "complete" ]; then
             ((jobscompleted++))
         fi
     done
+    echo "Jobs completed: ${jobscompleted}"
 done
 
 

From 8e151a31d30b939b67ef1b1a607c4aacc29ca07a Mon Sep 17 00:00:00 2001
From: Baxter Rogers <baxter.rogers@vanderbilt.edu>
Date: Thu, 25 Apr 2024 07:46:24 -0500
Subject: [PATCH 3/4] echo

---
 bin/supplemental_tools/dax-testrun.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/supplemental_tools/dax-testrun.sh b/bin/supplemental_tools/dax-testrun.sh
index 0579b375..942ce134 100755
--- a/bin/supplemental_tools/dax-testrun.sh
+++ b/bin/supplemental_tools/dax-testrun.sh
@@ -119,7 +119,7 @@ while [ "${jobscompleted}" -lt "${jobcount}" ]; do
             ((jobscompleted++))
         fi
     done
-    echo "Jobs completed: ${jobscompleted}"
+    echo "Jobs completed: ${jobscompleted} of ${jobcount}"
 done
 
 

From cb6fbac289cacacbbffc1e486bd058df25cdfee0 Mon Sep 17 00:00:00 2001
From: Baxter Rogers <baxter.rogers@vanderbilt.edu>
Date: Thu, 25 Apr 2024 11:54:51 -0500
Subject: [PATCH 4/4] d

---
 bin/supplemental_tools/dax-testrun.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/supplemental_tools/dax-testrun.sh b/bin/supplemental_tools/dax-testrun.sh
index 942ce134..03335da5 100755
--- a/bin/supplemental_tools/dax-testrun.sh
+++ b/bin/supplemental_tools/dax-testrun.sh
@@ -115,7 +115,7 @@ while [ "${jobscompleted}" -lt "${jobcount}" ]; do
         jobstate=$(rtracejob ${job} |grep "State")
         jobstate=$(echo ${jobstate##*|})
         echo "Job ${job} state: ${jobstate}"
-        if [ "${jobstate}" = "complete" ]; then
+        if [ "${jobstate}" = "completed" ]; then
             ((jobscompleted++))
         fi
     done