From 1d596d2914d539a6b2201ecae3895ed3ee2c1592 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Mon, 4 Sep 2023 16:20:00 -0500
Subject: [PATCH 01/31] add autobuild on delta

---
 .github/workflows/autobuild.yml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 .github/workflows/autobuild.yml

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
new file mode 100644
index 0000000000..8952952ab2
--- /dev/null
+++ b/.github/workflows/autobuild.yml
@@ -0,0 +1,23 @@
+name: Autobuild
+
+on:
+  schedule:
+    - cron: '15 06 * * *' # UTC 6:15am, corresponds to 00:15 CST or 01:15 CDT
+
+jobs:
+  Delta:
+    timeout-minutes: 60
+
+    runs-on: delta
+    name: Delta
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: autobuild
+    - name: build
+      run: |
+        echo "Running autobuild on delta"
+        hostname
+        uname -a
+        lsb_release -a

From 42d18d99103faa3e878881384118e09971a7f904 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Mon, 4 Sep 2023 16:28:33 -0500
Subject: [PATCH 02/31] add a test

---
 .github/workflows/autobuild.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 8952952ab2..9ebbd5ccf6 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -2,7 +2,9 @@ name: Autobuild
 
 on:
   schedule:
-    - cron: '15 06 * * *' # UTC 6:15am, corresponds to 00:15 CST or 01:15 CDT
+    # - cron: '15 06 * * *' # UTC 6:15am, corresponds to 00:15 CST or 01:15 CDT
+    # FIXME: For testing:
+    - cron: '30 04 * * *'
 
 jobs:
   Delta:

From 45e1409297a61e86ca187e8f9e5cfe032eb9dfbe Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Mon, 4 Sep 2023 16:30:11 -0500
Subject: [PATCH 03/31] cancel concurrent

---
 .github/workflows/autobuild.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 9ebbd5ccf6..08e8e1bcba 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -6,6 +6,13 @@ on:
     # FIXME: For testing:
     - cron: '30 04 * * *'
 
+
+# Cancel in progress CI runs when a new run targeting the same PR or branch/tag is triggered.
+# https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
 jobs:
   Delta:
     timeout-minutes: 60

From e9e6ee024f9b92fd3c49e21af55df7d1d2cd10b8 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Mon, 4 Sep 2023 16:40:03 -0500
Subject: [PATCH 04/31] on push

---
 .github/workflows/autobuild.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 08e8e1bcba..80c6703196 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -2,9 +2,10 @@ name: Autobuild
 
 on:
   schedule:
-    # - cron: '15 06 * * *' # UTC 6:15am, corresponds to 00:15 CST or 01:15 CDT
-    # FIXME: For testing:
-    - cron: '30 04 * * *'
+    - cron: '15 06 * * *' # UTC 6:15am, corresponds to 00:15 CST or 01:15 CDT
+  push:
+    paths:
+      - '.github/workflows/autobuild.yml'
 
 
 # Cancel in progress CI runs when a new run targeting the same PR or branch/tag is triggered.

From 560764f12953ede66347a677f812c0c6a3c00a20 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Wed, 11 Oct 2023 13:30:08 -0500
Subject: [PATCH 05/31] remove old ref

---
 .github/workflows/autobuild.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 80c6703196..fe7154148c 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -23,8 +23,6 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-      with:
-        ref: autobuild
     - name: build
       run: |
         echo "Running autobuild on delta"

From 11a401305943ad2d6d24a724869b95eb6ece4a67 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Wed, 11 Oct 2023 13:46:07 -0500
Subject: [PATCH 06/31] actually run a build, more comments

---
 .github/workflows/autobuild.yml | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index fe7154148c..6299bedf21 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -2,9 +2,11 @@ name: Autobuild
 
 on:
   schedule:
+    # Run the build as part of a fixed nightly schedule
     - cron: '15 06 * * *' # UTC 6:15am, corresponds to 00:15 CST or 01:15 CDT
   push:
     paths:
+      # Also run the build when this file gets modified as part of a PR
       - '.github/workflows/autobuild.yml'
 
 
@@ -19,13 +21,21 @@ jobs:
     timeout-minutes: 60
 
     runs-on: delta
-    name: Delta
+    name: Delta mpi-linux-x86_64  # Could test various builds (e.g., MPI, UCX, ...) via a build matrix
 
     steps:
     - uses: actions/checkout@v3
-    - name: build
+    - name: Host info
       run: |
         echo "Running autobuild on delta"
         hostname
         uname -a
         lsb_release -a
+    - name: build
+      run: |
+        cd ~/autobuild
+        ./build_local mpi-linux-x86_64
+    - name: results
+      run: |
+        cat ~/autobuild/messages/mpi-linux-x86_64
+        # Could add email notification here, e.g. via https://github.com/marketplace/actions/send-email

From bbc78d5de46c9aa75d2a0f0637dc915a94010c12 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 11:18:29 -0500
Subject: [PATCH 07/31] slurm script to submit autobuild run

---
 .github/workflows/delta-sbatch-slurm.sh | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 .github/workflows/delta-sbatch-slurm.sh

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
new file mode 100644
index 0000000000..e087443564
--- /dev/null
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -0,0 +1,20 @@
+#!/bin/bash -l
+#SBATCH -N 2
+#SBATCH -n 32
+#SBATCH -t 02:00:00
+#SBATCH -J autobuild
+#SBATCH -o $output
+#SBATCH -e $output
+#SBATCH -p regular
+#SBATCH --partition=cpu
+#SBATCH -A mzu-delta-cpu
+cd $indir
+cd $testdir
+module load libfabric; module load cmake
+#$make clean
+$make -C ../tests $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+$make -C ../examples $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+$make -C ../benchmarks $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+# Save make exit status
+status=\$?
+echo \$? > $result

From 6bd105a937f1614bd3d8e2a5c15c355010469b84 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 11:35:18 -0500
Subject: [PATCH 08/31] switch to interactive

---
 .github/workflows/delta-sbatch-slurm.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index e087443564..0b1fce5cfc 100644
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -6,9 +6,11 @@
 #SBATCH -o $output
 #SBATCH -e $output
 #SBATCH -p regular
-#SBATCH --partition=cpu
+#SBATCH --partition=cpu-interactive
+#SBATCH --pty bash
 #SBATCH -A mzu-delta-cpu
 cd $indir
+./build LIBS -j8 $target $flags
 cd $testdir
 module load libfabric; module load cmake
 #$make clean

From f0eb63f81e36f5921c75f4e2a8b150649fc58111 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 11:40:55 -0500
Subject: [PATCH 09/31] switch to all-test

---
 .github/workflows/delta-sbatch-slurm.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index 0b1fce5cfc..74689e6dd4 100644
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -9,14 +9,14 @@
 #SBATCH --partition=cpu-interactive
 #SBATCH --pty bash
 #SBATCH -A mzu-delta-cpu
-cd $indir
-./build LIBS -j8 $target $flags
-cd $testdir
+#cd $indir
 module load libfabric; module load cmake
+./build all-test -j8 $target $flags
+#cd $testdir
 #$make clean
-$make -C ../tests $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
-$make -C ../examples $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
-$make -C ../benchmarks $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+#$make -C ../tests $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+#$make -C ../examples $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+#$make -C ../benchmarks $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
 # Save make exit status
-status=\$?
-echo \$? > $result
+#status=\$?
+#echo \$? > $result

From d914b096f5c4efe0947228e4a4d6720ecd9002df Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 11:45:38 -0500
Subject: [PATCH 10/31] slimmed down script

---
 .github/workflows/delta-sbatch-slurm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 .github/workflows/delta-sbatch-slurm.sh

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
old mode 100644
new mode 100755
index 74689e6dd4..5c83074d23
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -11,7 +11,7 @@
 #SBATCH -A mzu-delta-cpu
 #cd $indir
 module load libfabric; module load cmake
-./build all-test -j8 $target $flags
+./build all-test $target -j8 $flags
 #cd $testdir
 #$make clean
 #$make -C ../tests $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts

From 6fddd49c8910eaeced2f64e71b9a400d342bcc83 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Fri, 3 Nov 2023 11:47:45 -0500
Subject: [PATCH 11/31] use srun directly

---
 .github/workflows/autobuild.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 6299bedf21..01141778ad 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -27,15 +27,19 @@ jobs:
     - uses: actions/checkout@v3
     - name: Host info
       run: |
+        set -x
         echo "Running autobuild on delta"
         hostname
         uname -a
         lsb_release -a
+        pwd
     - name: build
       run: |
-        cd ~/autobuild
-        ./build_local mpi-linux-x86_64
+        set -x
+        export target="mpi-linux-x86_64"
+        srun -A mzu-delta-cpu --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
-        cat ~/autobuild/messages/mpi-linux-x86_64
+        # this is incorrect now:
+        ## cat ~/autobuild/messages/mpi-linux-x86_64
         # Could add email notification here, e.g. via https://github.com/marketplace/actions/send-email

From b33e7b35bd25d153486ed61a847a21761328aeed Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Fri, 3 Nov 2023 11:59:39 -0500
Subject: [PATCH 12/31] bump nnodes, ntasks, mem

---
 .github/workflows/autobuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 01141778ad..2108a98601 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -37,7 +37,7 @@ jobs:
       run: |
         set -x
         export target="mpi-linux-x86_64"
-        srun -A mzu-delta-cpu --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
+        srun -A mzu-delta-cpu --mem=16g --nodes=2 --tasks=128 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
         # this is incorrect now:

From ea44037e6ee412213f42e1e623c915c45d709923 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Fri, 3 Nov 2023 12:01:26 -0500
Subject: [PATCH 13/31] remove ntasks

---
 .github/workflows/autobuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 2108a98601..6f6ab73a9d 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -37,7 +37,7 @@ jobs:
       run: |
         set -x
         export target="mpi-linux-x86_64"
-        srun -A mzu-delta-cpu --mem=16g --nodes=2 --tasks=128 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
+        srun -A mzu-delta-cpu --mem=16g --nodes=2 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
         # this is incorrect now:

From c1db5dce3d65670551d4608239424562eeb137b6 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 12:03:09 -0500
Subject: [PATCH 14/31] proper production and error checking flags in build

---
 .github/workflows/delta-sbatch-slurm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index 5c83074d23..fd6f329618 100755
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -11,7 +11,7 @@
 #SBATCH -A mzu-delta-cpu
 #cd $indir
 module load libfabric; module load cmake
-./build all-test $target -j8 $flags
+./build all-test $target --with-production --enable-error-checking -j16 -g
 #cd $testdir
 #$make clean
 #$make -C ../tests $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts

From b55b653cbed7a0c67d6c0b265740c7d7c7a904ed Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 12:10:44 -0500
Subject: [PATCH 15/31] more verbosity in the script

---
 .github/workflows/delta-sbatch-slurm.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index fd6f329618..c70582297f 100755
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -10,6 +10,7 @@
 #SBATCH --pty bash
 #SBATCH -A mzu-delta-cpu
 #cd $indir
+set -x
 module load libfabric; module load cmake
 ./build all-test $target --with-production --enable-error-checking -j16 -g
 #cd $testdir

From 325221ff22c16fe10a1ad393f64066a7cc16253f Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Fri, 3 Nov 2023 12:12:32 -0500
Subject: [PATCH 16/31] bump cpus-per-task

---
 .github/workflows/autobuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 6f6ab73a9d..cd5f3073d0 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -37,7 +37,7 @@ jobs:
       run: |
         set -x
         export target="mpi-linux-x86_64"
-        srun -A mzu-delta-cpu --mem=16g --nodes=2 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
+        srun -A mzu-delta-cpu --cpus-per-task=128 --mem=16g --nodes=2 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
         # this is incorrect now:

From 05a93a9e6d4f9d224760e615d0b6f656dcfebc9b Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Fri, 3 Nov 2023 12:14:40 -0500
Subject: [PATCH 17/31] temporarily remove Github CI files

---
 .github/workflows/buildold.yml           | 20 --------
 .github/workflows/changa.yml             | 23 ---------
 .github/workflows/charm4py.yml           | 33 -------------
 .github/workflows/cuda.yml               | 22 ---------
 .github/workflows/doc.yml                | 28 -----------
 .github/workflows/mpi_linux_smp.yml      | 22 ---------
 .github/workflows/multicore_darwin.yml   | 27 ----------
 .github/workflows/namd.yml               | 63 ------------------------
 .github/workflows/netlrts_darwin.yml     | 19 -------
 .github/workflows/netlrts_linux_i386.yml | 24 ---------
 .github/workflows/shellcheck.yml         | 27 ----------
 .github/workflows/spack.yml              | 51 -------------------
 .github/workflows/syncft_mpi.yml         | 18 -------
 .github/workflows/syncft_netlrts.yml     | 16 ------
 .github/workflows/ucx.yml                | 48 ------------------
 .github/workflows/verbs.yml              | 35 -------------
 .github/workflows/windows.yml            | 29 -----------
 17 files changed, 505 deletions(-)
 delete mode 100644 .github/workflows/buildold.yml
 delete mode 100644 .github/workflows/changa.yml
 delete mode 100644 .github/workflows/charm4py.yml
 delete mode 100644 .github/workflows/cuda.yml
 delete mode 100644 .github/workflows/doc.yml
 delete mode 100644 .github/workflows/mpi_linux_smp.yml
 delete mode 100644 .github/workflows/multicore_darwin.yml
 delete mode 100644 .github/workflows/namd.yml
 delete mode 100644 .github/workflows/netlrts_darwin.yml
 delete mode 100644 .github/workflows/netlrts_linux_i386.yml
 delete mode 100644 .github/workflows/shellcheck.yml
 delete mode 100644 .github/workflows/spack.yml
 delete mode 100644 .github/workflows/syncft_mpi.yml
 delete mode 100644 .github/workflows/syncft_netlrts.yml
 delete mode 100644 .github/workflows/ucx.yml
 delete mode 100644 .github/workflows/verbs.yml
 delete mode 100644 .github/workflows/windows.yml

diff --git a/.github/workflows/buildold.yml b/.github/workflows/buildold.yml
deleted file mode 100644
index 244cc26270..0000000000
--- a/.github/workflows/buildold.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-name: NetLRTS Linux buildold
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 90
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: build
-      run: ./buildold all-test netlrts-linux-x86_64 -g -j4 --with-production
-    - name: test
-      run: make -C netlrts-linux-x86_64/tmp test TESTOPTS="++local"
-    - name: testp P=2
-      run: make -C netlrts-linux-x86_64/tmp testp TESTOPTS="++local" P=2
-    - name: testp P=3
-      run: make -C netlrts-linux-x86_64/tmp testp TESTOPTS="++local" P=3
diff --git a/.github/workflows/changa.yml b/.github/workflows/changa.yml
deleted file mode 100644
index d3d3bd6762..0000000000
--- a/.github/workflows/changa.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: ChaNGa
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: build-charm++
-      run: ./build ChaNGa netlrts-linux-x86_64 -g -j4 --with-production
-    - name: build-changa
-      run: |
-        git clone https://github.com/N-bodyshop/utility
-        git clone https://github.com/N-bodyshop/changa
-        cd changa
-        ./configure
-        make -j4
-    - name: test-changa
-      run: |
-        cd changa/teststep
-        make test
diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml
deleted file mode 100644
index f401f6beab..0000000000
--- a/.github/workflows/charm4py.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-name: Charm4py
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: build-charm++
-      run: |
-        git fetch --unshallow # Need full repo for 'git describe' used by charm4py
-        ./build charm4py netlrts-linux-x86_64 tcp -g -j4 --with-production
-    - name: build-charm4py
-      run: |
-        pip3 install setuptools 'cython<3' cffi greenlet numpy
-        git clone https://github.com/UIUC-PPL/charm4py
-        export PYTHONPATH="$PWD/charm4py"
-
-        mkdir -p charm4py/charm_src/charm/bin && cp bin/charmrun charm4py/charm_src/charm/bin
-        mkdir -p charm4py/charm_src/charm/lib && cp lib/libcharm.so charm4py/charm_src/charm/lib
-        mkdir -p charm4py/charm_src/charm/include && cp -r -L include/* charm4py/charm_src/charm/include
-
-        cd charm4py
-        export CHARM4PY_BUILD_CFFI=1
-        python3 setup.py build_ext --inplace
-    - name: test-charm4py
-      run: |
-        export PYTHONPATH="$PWD/charm4py"
-        export CHARM4PY_TEST_NUM_PROCESSES=2
-        cd charm4py
-        python3 auto_test.py
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
deleted file mode 100644
index 1b921c3cdb..0000000000
--- a/.github/workflows/cuda.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: NetLRTS Linux CUDA buildonly
-
-# Buildonly test, as CUDA needs an actual device to run.
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 45
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: build
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install nvidia-cuda-toolkit
-        ./build all-test netlrts-linux-x86_64 cuda -j4 -g
-        export CUDATOOLKIT_HOME=/usr # TODO: make this unnecessary
-        make -j4 -C netlrts-linux-x86_64-cuda/examples/charm++/cuda OPTS="-g"
-        make -j4 -C netlrts-linux-x86_64-cuda/examples/ampi/cuda OPTS="-g"
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
deleted file mode 100644
index 866616fdf5..0000000000
--- a/.github/workflows/doc.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-name: Sphinx-doc
-
-# Check if documentation builds with sphinx. Only triggered if a file
-# in doc/ is changed.
-
-on:
-  merge_group:
-  pull_request:
-    paths: 
-      - 'doc/**'
-
-jobs:
-  sphinx-build:
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    -
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-    - name: Install prerequisites
-      run: pip3 install sphinx sphinx_rtd_theme        
-    - name: Run sphinx
-      run: |
-        cd doc
-        make html
diff --git a/.github/workflows/mpi_linux_smp.yml b/.github/workflows/mpi_linux_smp.yml
deleted file mode 100644
index 9bfede622e..0000000000
--- a/.github/workflows/mpi_linux_smp.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: MPI Linux SMP
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 60
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: install-prerequisites
-      run: sudo apt-get update && sudo apt-get install -y mpich libmpich-dev
-    - name: build
-      run: ./build all-test mpi-linux-x86_64 smp --build-shared --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C mpi-linux-x86_64-smp/tmp test TESTOPTS="+setcpuaffinity"
-    # Disabled due to several hangs and segfaults.
-    #- name: testp
-    #  run: make -C mpi-linux-x86_64-smp/tmp testp P=4 TESTOPTS="+setcpuaffinity ++ppn 2"
diff --git a/.github/workflows/multicore_darwin.yml b/.github/workflows/multicore_darwin.yml
deleted file mode 100644
index 31d2c3f0e9..0000000000
--- a/.github/workflows/multicore_darwin.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-name: Multicore Darwin tracing
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 60
-
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: build
-      run: ./build LIBS multicore-darwin-x86_64 -g -j3 --with-production --enable-tracing
-    - name: test
-      run: |
-        make -C multicore-darwin-x86_64/tmp all-test -j3 OPTS="-g -tracemode projections"
-        make -C multicore-darwin-x86_64/tmp test
-    - name: projections
-      run: |
-        git clone https://github.com/UIUC-PPL/projections
-        cd projections
-        make
-        proj=$PWD/bin/projections
-        cd ..
-        files=$(find . -name *.sts)
-        for f in $files; do echo $f; pushd .; cd $(dirname $f); $proj --exit $(basename $f); popd; done
diff --git a/.github/workflows/namd.yml b/.github/workflows/namd.yml
deleted file mode 100644
index f7bb08998e..0000000000
--- a/.github/workflows/namd.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-name: NAMD
-
-on: [pull_request, merge_group]
-
-jobs:
-  # Not named 'build' since 'build' is used by our CI for builds that must succeed before merge.
-  # Since NAMD requires a secret to download its source from Gitlab, CI builds from outside PPl
-  # always fail in this test since the secret is not available.
-  build-and-test-namd:
-    env:
-        namd_secret: ${{ secrets.NAMD_CI_USERNAME_TOKEN }}
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: install-prerequisites
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install sfftw-dev
-        # test build without Fortran support
-        sudo apt-get -y remove gfortran
-    - name: build-charm++
-      if: ${{ env.namd_secret != '' }}
-      run: ./build LIBS netlrts-linux-x86_64 -g -j4 --with-production
-    - name: build-namd
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        cd ..
-        git clone https://${{ secrets.NAMD_CI_USERNAME_TOKEN }}@gitlab.com/tcbgUIUC/namd.git
-        cd namd
-        ./config Linux-x86_64-g++ --charm-base ../charm --charm-arch netlrts-linux-x86_64 --without-tcl
-        cd Linux-x86_64-g++
-        make -j4
-    - name: test-namd-alanin
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        cd ../namd/Linux-x86_64-g++
-        ./charmrun ++local +p2 ./namd2 src/alanin
-
-    - name: Cache apoa1 files
-      if: ${{ env.namd_secret != '' }}
-      id: cache-apoa1-files
-      uses: actions/cache@v2.0.0
-      with:
-        path: ~/namddata/apoa1
-        key: namd-apoa1-files
-    - name: Download apoa1 files
-      if: steps.cache-apoa1-files.outputs.cache-hit != 'true'
-      run: |
-        mkdir -p ~/namddata
-        cd ~/namddata
-        wget http://www.ks.uiuc.edu/Research/namd/utilities/apoa1.tar.gz
-        tar xzf apoa1.tar.gz
-        sed -i 's,500,250,' apoa1/apoa1.namd # Reduce number of steps
-        sed -i 's,/usr/tmp,/tmp,' apoa1/apoa1.namd # Change output prefix
-        rm apoa1.tar.gz
-
-    - name: test-namd-apoa1
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        cd ../namd/Linux-x86_64-g++
-        ./charmrun ++local +p4 ./namd2 ~/namddata/apoa1/apoa1.namd
diff --git a/.github/workflows/netlrts_darwin.yml b/.github/workflows/netlrts_darwin.yml
deleted file mode 100644
index 73f7167bd3..0000000000
--- a/.github/workflows/netlrts_darwin.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-name: NetLRTS Darwin
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 60
-
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: build
-      run: ./build all-test netlrts-darwin-x86_64 --build-shared --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C netlrts-darwin-x86_64/tmp test TESTOPTS="++local"
-    - name: testp
-      run: make -C netlrts-darwin-x86_64/tmp testp P=2 TESTOPTS="++local"
diff --git a/.github/workflows/netlrts_linux_i386.yml b/.github/workflows/netlrts_linux_i386.yml
deleted file mode 100644
index 49f2b87c5e..0000000000
--- a/.github/workflows/netlrts_linux_i386.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-name: NetLRTS Linux i386
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 60
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: install-prerequisites
-      run: |
-        sudo dpkg --add-architecture i386
-        sudo apt-get update
-        sudo apt-get install -y gcc-multilib g++-multilib gfortran-multilib zlib1g-dev:i386 libjpeg-dev:i386
-    - name: build
-      run: ./build all-test netlrts-linux-i386 --build-shared --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C netlrts-linux-i386/tmp test TESTOPTS="++local"
-    - name: testp
-      run: make -C netlrts-linux-i386/tmp testp P=2 TESTOPTS="++local"
diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml
deleted file mode 100644
index ff175a4994..0000000000
--- a/.github/workflows/shellcheck.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-name: ShellCheck
-
-# See https://github.com/koalaman/shellcheck/wiki
-# for explanations of ShellCheck error codes
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 45
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: ShellCheck
-      run: |
-        # TODO: add more scripts
-        shellcheck \
-          build \
-          buildcmake \
-          package-tarball.sh \
-          relink.script \
-          src/scripts/commitid.sh \
-          src/scripts/testrun \
-          src/arch/multicore/charmrun \
-          src/arch/mpi-win-x86_64/charmrun \
diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
deleted file mode 100644
index b9bbf03e48..0000000000
--- a/.github/workflows/spack.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-name: Spack
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    name: ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04, macos-latest]
-
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Install Spack
-      run: |
-        cd ..
-        git clone --depth 1 https://github.com/spack/spack
-        cd spack
-        [[ $(uname) == "Linux" ]] && sudo apt-get install -y gfortran gfortran-7 gfortran-8 gfortran-9 gfortran-10
-
-        # Use this branch for testing, not main
-        sed -i -e 's,="main",="${{ github.head_ref }}",' var/spack/repos/builtin/packages/charmpp/package.py
-
-        # Use a fork, not the main repo
-        # If in merge_group mode, the branch should be created in the main repo
-        if [[ ${{ github.event_name }} == 'pull_request' ]]; then
-          sed -i -e 's,UIUC-PPL/charm.git,${{github.event.pull_request.head.repo.full_name}},' var/spack/repos/builtin/packages/charmpp/package.py
-        fi
-
-        # Compile with debug symbols
-        sed -i -e 's,build(target,options.append("-g"); build(target,' var/spack/repos/builtin/packages/charmpp/package.py
-
-        # Add +setcpuaffinity option to TESTOPTS
-        sed -i -e 's,\+\+local,++local +setcpuaffinity,' var/spack/repos/builtin/packages/charmpp/package.py
-
-        # No need for automake/autoconf
-        sed -i -e '/automake/d' var/spack/repos/builtin/packages/charmpp/package.py
-        sed -i -e '/autoconf/d' var/spack/repos/builtin/packages/charmpp/package.py
-
-    - name: Build Charm++
-      run: |
-        cd ../spack
-        source share/spack/setup-env.sh
-        spack compiler find
-        spack compiler list
-        spack install --test root charmpp@main
diff --git a/.github/workflows/syncft_mpi.yml b/.github/workflows/syncft_mpi.yml
deleted file mode 100644
index 27f27f8878..0000000000
--- a/.github/workflows/syncft_mpi.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-name: MPI Linux SyncFT
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 90
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: install-prerequisites
-      run: sudo apt-get update && sudo apt-get install -y mpich libmpich-dev
-    - name: build
-      run: ./build all-test mpi-linux-x86_64 syncft -j2 -g
-    - name: test
-      run: make -C mpi-linux-x86_64-syncft/tmp mpisyncfttest
diff --git a/.github/workflows/syncft_netlrts.yml b/.github/workflows/syncft_netlrts.yml
deleted file mode 100644
index c2157d7040..0000000000
--- a/.github/workflows/syncft_netlrts.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-name: NetLRTS Linux SyncFT
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 90
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: build
-      run: ./build all-test netlrts-linux-x86_64 syncft -j2 -g
-    - name: test
-      run: make -C netlrts-linux-x86_64-syncft/tmp syncfttest TESTOPTS="++local"
diff --git a/.github/workflows/ucx.yml b/.github/workflows/ucx.yml
deleted file mode 100644
index 121c9f1be5..0000000000
--- a/.github/workflows/ucx.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: UCX Linux
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 60
-
-    runs-on: ubuntu-22.04
-
-    steps:
-    - uses: actions/checkout@v1
-    # Uncomment the lines below to set up a tmate session for debugging.
-    # See https://github.com/marketplace/actions/debugging-with-tmate for details.
-    # This can't be enabled all the time as the tmate session will wait for a user to connect before running
-    # the build.
-    #- name: Tmate session for debugging
-    #  uses: mxschmitt/action-tmate@v2
-    - name: install-prerequisites
-      run: |
-        sudo apt-get update
-        sudo apt-get install libucx-dev libevent-dev libhwloc-dev
-    - name: build-openpmix
-      run: |
-        wget https://github.com/openpmix/openpmix/releases/download/v3.1.5/pmix-3.1.5.tar.gz
-        tar -xf pmix-3.1.5.tar.gz
-        cd pmix-3.1.5
-        ./configure
-        make -j4
-        sudo make install
-    # OpenMPI is needed to launch Charm++ programs with the UCX backend
-    - name: build-ompi
-      run: |
-        wget https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.5.tar.gz
-        tar -xf openmpi-4.0.5.tar.gz
-        cd openmpi-4.0.5
-        ./configure --enable-mca-no-build=btl-uct --with-pmix=/usr/local
-        make -j4
-        sudo make install
-    - name: build
-      run: ./build all-test ucx-linux-x86_64 openpmix -g -j4 --with-production
-    - name: test
-      run: |
-        export PATH=/usr/local/bin:$PATH
-        export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
-        export OMPI_MCA_rmaps_base_oversubscribe=1
-        make -C ucx-linux-x86_64-openpmix/tests test
-        make -C ucx-linux-x86_64-openpmix/examples test
diff --git a/.github/workflows/verbs.yml b/.github/workflows/verbs.yml
deleted file mode 100644
index de709936f8..0000000000
--- a/.github/workflows/verbs.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Verbs Linux Buildonly Tarball
-
-# Buildonly test, as the machine layer needs an actual device to run.
-# Also test packaging a tarball and building from it.
-# Since it is buildonly, test both build and buildold with the tarball.
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 45
-
-    runs-on: ubuntu-20.04
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: install-prerequisites
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install libibverbs-dev
-    - name: package-tarball
-      run: |
-        ./package-tarball.sh
-        cd ..
-        tar -xzf charm/charm-*.tar.gz
-    - name: build-smp
-      run: |
-        cd ../charm-*
-        # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-        ./build all-test verbs-linux-x86_64 smp -j4 -g --build-shared
-    - name: buildold-nonsmp
-      run: |
-        cd ../charm-*
-        # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-        ./buildold all-test verbs-linux-x86_64 -j4 -g --build-shared
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
deleted file mode 100644
index bc753b5495..0000000000
--- a/.github/workflows/windows.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-name: NetLRTS Windows
-
-on: [pull_request, merge_group]
-
-jobs:
-  build:
-    timeout-minutes: 90
-
-    runs-on: windows-latest
-
-    defaults:
-      run:
-        shell: msys2 {0}
-
-    steps:
-    - uses: actions/checkout@v2
-    - uses: msys2/setup-msys2@v2
-      with:
-        msystem: MINGW64
-        update: true
-        install: git mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-cmake make libtool autoconf automake
-    - name: build
-      run: |
-        . src/arch/win/vsenv.sh
-        ./build AMPI netlrts-win-x86_64 --with-production -j2
-        make -C netlrts-win-x86_64/tests -j2
-    - name: test
-      run: |
-        make -C netlrts-win-x86_64/tests test TESTOPTS="++local"

From f9b623594740fcceaf4d8e56a1a836a321a8bb26 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Fri, 3 Nov 2023 12:15:27 -0500
Subject: [PATCH 18/31] nnodes=1

---
 .github/workflows/autobuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index cd5f3073d0..717a582a9c 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -37,7 +37,7 @@ jobs:
       run: |
         set -x
         export target="mpi-linux-x86_64"
-        srun -A mzu-delta-cpu --cpus-per-task=128 --mem=16g --nodes=2 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
+        srun -A mzu-delta-cpu --cpus-per-task=128 --mem=16g --nodes=1 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
         # this is incorrect now:

From 253c1f89ff3e52d63c27d93a1107d96e1194a792 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 12:25:43 -0500
Subject: [PATCH 19/31] run the tests after compiling them

---
 .github/workflows/delta-sbatch-slurm.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index c70582297f..0e4414ce17 100755
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -1,6 +1,5 @@
 #!/bin/bash -l
-#SBATCH -N 2
-#SBATCH -n 32
+#SBATCH -N 1
 #SBATCH -t 02:00:00
 #SBATCH -J autobuild
 #SBATCH -o $output
@@ -13,6 +12,7 @@
 set -x
 module load libfabric; module load cmake
 ./build all-test $target --with-production --enable-error-checking -j16 -g
+make -C $target/tmp test TESTOPTS+="+setcpuaffinity"
 #cd $testdir
 #$make clean
 #$make -C ../tests $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts

From d93f40c5a9430c0c4a49f4a4e044c91ce4344a69 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Fri, 3 Nov 2023 12:33:04 -0500
Subject: [PATCH 20/31] add delta sbatch as run dependency

---
 .github/workflows/autobuild.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 717a582a9c..a562a84347 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -8,6 +8,7 @@ on:
     paths:
       # Also run the build when this file gets modified as part of a PR
       - '.github/workflows/autobuild.yml'
+      - '.github/workflows/delta-sbatch-slurm.sh'
 
 
 # Cancel in progress CI runs when a new run targeting the same PR or branch/tag is triggered.

From 5ee524b5c3d44d12575d46fa435e282804c2254a Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 3 Nov 2023 15:05:12 -0500
Subject: [PATCH 21/31] switch back to sbatch approach

---
 .github/workflows/delta-sbatch-slurm.sh | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index 0e4414ce17..4e6e0102b1 100755
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -1,17 +1,14 @@
 #!/bin/bash -l
-#SBATCH -N 1
-#SBATCH -t 02:00:00
+#SBATCH -N 2
+#SBATCH -n 32
+#SBATCH -t 0:30:00
 #SBATCH -J autobuild
-#SBATCH -o $output
-#SBATCH -e $output
-#SBATCH -p regular
-#SBATCH --partition=cpu-interactive
-#SBATCH --pty bash
+#SBATCH -p cpu
 #SBATCH -A mzu-delta-cpu
 #cd $indir
 set -x
 module load libfabric; module load cmake
-./build all-test $target --with-production --enable-error-checking -j16 -g
+./build all-test $target --with-production --enable-error-checking -j -g
 make -C $target/tmp test TESTOPTS+="+setcpuaffinity"
 #cd $testdir
 #$make clean

From 0a62264849b8ece6740710ab4ebd005d7a6546e9 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 10 Nov 2023 13:26:31 -0600
Subject: [PATCH 22/31] script to poll the jobqueue and then cat the results of
 the run

---
 .github/workflows/jobmonitor.sh | 68 +++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100755 .github/workflows/jobmonitor.sh

diff --git a/.github/workflows/jobmonitor.sh b/.github/workflows/jobmonitor.sh
new file mode 100755
index 0000000000..75e58cbdfb
--- /dev/null
+++ b/.github/workflows/jobmonitor.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+export script=$1;
+queue_qsub=sbatch
+queue_kill=scancel
+queue_stat="squeue -j"
+
+End() {
+        echo "autobuild> $queue_kill $jobid ..."
+        $queue_kill $jobid
+        exit $1
+}
+echo "Submitting batch job for>  $target OPTS=$flags"
+echo " using the command> $queue_qsub $script"
+chmod 755 $script
+while [ -z "$jobid" ]
+do
+    $queue_qsub $script > .status.$$ 2>&1
+    if grep 'have no accounts' .status.$$ > /dev/null
+    then
+        echo "NO account for submitting batch job!"
+        rm -f .status.$$
+        exit 1
+    fi
+    jobid=`cat .status.$$ | tail -1 | awk '{print $4}'`
+    rm -f .status.$$
+done
+echo "Job enqueued under job ID $jobid"
+export output=$jobid.output
+export result=$jobid.result
+# kill job if interrupted
+trap 'End 1' 2 3
+retry=0
+# Wait for the job to complete, by checking its status
+while [ true ]
+do
+    $queue_stat $jobid > tmp.$$
+    linecount=`wc -l tmp.$$ | awk '{print $1}' `
+    exitstatus=$?
+    #if [[ $exitstatus != 0 || $linecount != 2 ]]
+    if [[ $linecount != 2 ]]
+    then
+	# The job is done-- print its output
+        rm tmp.$$
+	# When job hangs, result file does not exist
+        test -f $result && status=`cat $result` || status=1
+        echo "==================================== OUTPUT (STDOUT & STDERR) ========================================"
+        cat $output
+        echo "======================================================================================================"
+        if [[ "$status" != 0 ]];
+        then
+            #print script
+            echo "=============================================== SCRIPT ==============================================="
+            cat $script
+            echo "======================================================================================================"
+            echo "=============================================== RESULT ==============================================="
+            cat $result
+            echo "======================================================================================================"
+        fi
+        # Delete result, script, output and error
+        rm -f $result $output
+        exit $status
+    fi
+# The job is still queued or running-- print status and wait
+    tail -1 tmp.$$
+    rm tmp.$$
+    sleep 60
+done

From b982df5c9f3c9f54c37cc131d2587426c27d80e5 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 10 Nov 2023 13:27:09 -0600
Subject: [PATCH 23/31] batch version revised to use SLURM variables for
 filenames

---
 .github/workflows/delta-sbatch-slurm.sh | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index 4e6e0102b1..12f38f6b77 100755
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -1,6 +1,8 @@
 #!/bin/bash -l
 #SBATCH -N 2
-#SBATCH -n 32
+#SBATCH -n 64
+#SBATCH -o %j.output
+#SBATCH -e %j.output
 #SBATCH -t 0:30:00
 #SBATCH -J autobuild
 #SBATCH -p cpu
@@ -8,13 +10,12 @@
 #cd $indir
 set -x
 module load libfabric; module load cmake
-./build all-test $target --with-production --enable-error-checking -j -g
-make -C $target/tmp test TESTOPTS+="+setcpuaffinity"
-#cd $testdir
-#$make clean
-#$make -C ../tests $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
-#$make -C ../examples $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
-#$make -C ../benchmarks $target OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+./build all-test $target --with-production --enable-error-checking -j64 -g
+#
+cd $target
+make -C tests test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+make -C examples test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+make -C benchmarks test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
 # Save make exit status
-#status=\$?
-#echo \$? > $result
+status=$?
+echo $status > ../$SLURM_JOBID.result

From 3596981743201ac3e45c2a1772b954e243110dd2 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Fri, 10 Nov 2023 13:28:14 -0600
Subject: [PATCH 24/31] revised to call the jobmonitor for resource montoring

---
 .github/workflows/autobuild.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index a562a84347..da74ba70cc 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -36,9 +36,9 @@ jobs:
         pwd
     - name: build
       run: |
-        set -x
+        set -ex
         export target="mpi-linux-x86_64"
-        srun -A mzu-delta-cpu --cpus-per-task=128 --mem=16g --nodes=1 --partition=cpu-interactive .github/workflows/delta-sbatch-slurm.sh
+        .github/workflows/jobmonitor.sh .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
         # this is incorrect now:

From 88cc925476f6ffb325f6f0cbe6862f7e4eb343f5 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Tue, 21 Nov 2023 14:03:30 -0600
Subject: [PATCH 25/31] preserve output and status in latest.output and
 latest.status increase job time to 1 hour to account for days when the
 filesystem is exceptionally sloooow

---
 .github/workflows/delta-sbatch-slurm.sh | 2 +-
 .github/workflows/jobmonitor.sh         | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index 12f38f6b77..af954d456e 100755
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -3,7 +3,7 @@
 #SBATCH -n 64
 #SBATCH -o %j.output
 #SBATCH -e %j.output
-#SBATCH -t 0:30:00
+#SBATCH -t 1:00:00
 #SBATCH -J autobuild
 #SBATCH -p cpu
 #SBATCH -A mzu-delta-cpu
diff --git a/.github/workflows/jobmonitor.sh b/.github/workflows/jobmonitor.sh
index 75e58cbdfb..f4604a8cae 100755
--- a/.github/workflows/jobmonitor.sh
+++ b/.github/workflows/jobmonitor.sh
@@ -57,8 +57,9 @@ do
             cat $result
             echo "======================================================================================================"
         fi
-        # Delete result, script, output and error
-        rm -f $result $output
+	# mv result and output to result.latest
+	mv $result result.latest
+	mv $output output.latest
         exit $status
     fi
 # The job is still queued or running-- print status and wait

From e28a55973f96740a9cdf18017167791be57c68ff Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Tue, 21 Nov 2023 14:10:36 -0600
Subject: [PATCH 26/31] revise to check status file and report success/failure

---
 .github/workflows/autobuild.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index da74ba70cc..8bf009076e 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -41,6 +41,10 @@ jobs:
         .github/workflows/jobmonitor.sh .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
-        # this is incorrect now:
-        ## cat ~/autobuild/messages/mpi-linux-x86_64
-        # Could add email notification here, e.g. via https://github.com/marketplace/actions/send-email
+      if grep '0' results.latest
+      then
+	echo "Success"
+      else
+	echo "Failure"
+      fi	
+      # should also https://github.com/marketplace/actions/send-email
\ No newline at end of file

From 07be593fa7e35303bd85022842d4e7449e48f3bc Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Tue, 21 Nov 2023 14:15:09 -0600
Subject: [PATCH 27/31] remove tabs that yaml hates

---
 .github/workflows/autobuild.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 8bf009076e..bf5f30746f 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -43,8 +43,8 @@ jobs:
       run: |
       if grep '0' results.latest
       then
-	echo "Success"
+        echo "Success"
       else
-	echo "Failure"
+        echo "Failure"
       fi	
       # should also https://github.com/marketplace/actions/send-email
\ No newline at end of file

From 92931ffb84ea410b6e306afeb5268d2aa752a5c3 Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Tue, 21 Nov 2023 14:17:51 -0600
Subject: [PATCH 28/31] more whitespace syntax adherence

---
 .github/workflows/autobuild.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index bf5f30746f..84606c705d 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -41,10 +41,10 @@ jobs:
         .github/workflows/jobmonitor.sh .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
-      if grep '0' results.latest
-      then
-        echo "Success"
-      else
-        echo "Failure"
-      fi	
-      # should also https://github.com/marketplace/actions/send-email
\ No newline at end of file
+        if grep '0' results.latest
+        then
+          echo "Success"
+        else
+          echo "Failure"
+        fi	
+        # should also https://github.com/marketplace/actions/send-email
\ No newline at end of file

From 7e085495a9a738c51e7b154a1fc09dced6c9be4f Mon Sep 17 00:00:00 2001
From: Eric Bohm <eric@hpccharm.com>
Date: Tue, 21 Nov 2023 14:58:56 -0600
Subject: [PATCH 29/31] !fixup typo in result file name

---
 .github/workflows/autobuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 84606c705d..93a2e10a27 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -41,7 +41,7 @@ jobs:
         .github/workflows/jobmonitor.sh .github/workflows/delta-sbatch-slurm.sh
     - name: results
       run: |
-        if grep '0' results.latest
+        if grep '0' result.latest
         then
           echo "Success"
         else

From 1eb8e7f29aefcd1b508b547779509c8826ed55ae Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Wed, 22 Nov 2023 17:29:14 -0600
Subject: [PATCH 30/31] disable normal CI

---
 .github/workflows/ci.yaml | 435 --------------------------------------
 1 file changed, 435 deletions(-)
 delete mode 100644 .github/workflows/ci.yaml

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
deleted file mode 100644
index c5a712296b..0000000000
--- a/.github/workflows/ci.yaml
+++ /dev/null
@@ -1,435 +0,0 @@
-name: CI
-
-on: [pull_request, merge_group]
-
-# Cancel in progress CI runs when a new run targeting the same PR or branch/tag is triggered.
-# https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  buildold_netlrts-linux-x86_64:
-    timeout-minutes: 90
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Run build
-      run: ./buildold all-test netlrts-linux-x86_64 -g -j4 --with-production
-    - name: test
-      run: make -C netlrts-linux-x86_64/tmp test TESTOPTS="++local"
-    - name: testp P=2
-      run: make -C netlrts-linux-x86_64/tmp testp TESTOPTS="++local" P=2
-    - name: testp P=3
-      run: make -C netlrts-linux-x86_64/tmp testp TESTOPTS="++local" P=3
-
-  changa_netlrts-linux-x86_64:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: build-charm++
-      run: ./build ChaNGa netlrts-linux-x86_64 -g -j4 --with-production
-    - name: build-changa
-      run: |
-        git clone https://github.com/N-bodyshop/utility
-        git clone https://github.com/N-bodyshop/changa
-        cd changa
-        ./configure
-        make -j4
-    - name: test-changa
-      run: |
-        cd changa/teststep
-        make test
-
-  charm4py_netlrts-linux-x86_64_tcp:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: build-charm++
-      run: |
-        git fetch --unshallow # Need full repo for 'git describe' used by charm4py
-        ./build charm4py netlrts-linux-x86_64 tcp -g -j4 --with-production
-    - name: build-charm4py
-      run: |
-        pip3 install setuptools 'cython<3' cffi greenlet numpy
-        git clone https://github.com/UIUC-PPL/charm4py
-        export PYTHONPATH="$PWD/charm4py"
-
-        mkdir -p charm4py/charm_src/charm/bin && cp bin/charmrun charm4py/charm_src/charm/bin
-        mkdir -p charm4py/charm_src/charm/lib && cp lib/libcharm.so charm4py/charm_src/charm/lib
-        mkdir -p charm4py/charm_src/charm/include && cp -r -L include/* charm4py/charm_src/charm/include
-
-        cd charm4py
-        export CHARM4PY_BUILD_CFFI=1
-        python3 setup.py build_ext --inplace
-    - name: test-charm4py
-      run: |
-        export PYTHONPATH="$PWD/charm4py"
-        export CHARM4PY_TEST_NUM_PROCESSES=2
-        cd charm4py
-        python3 auto_test.py
-
-  cuda_netlrts-linux-x86_64_buildonly:
-    # Buildonly test, as CUDA needs an actual device to run.
-    timeout-minutes: 45
-    runs-on: ubuntu-20.04  # FIXME: ubuntu-latest does not work
-    steps:
-    - uses: actions/checkout@v4
-    - name: build
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install nvidia-cuda-toolkit
-        ./build all-test netlrts-linux-x86_64 cuda -j4 -g
-        export CUDATOOLKIT_HOME=/usr # TODO: make this unnecessary
-        make -j4 -C netlrts-linux-x86_64-cuda/examples/charm++/cuda OPTS="-g"
-        make -j4 -C netlrts-linux-x86_64-cuda/examples/ampi/cuda OPTS="-g"
-
-  doc:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    -
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Install prerequisites
-      run: pip3 install sphinx sphinx_rtd_theme
-    - name: Run sphinx
-      run: |
-        cd doc
-        make html
-
-  mpi-linux-x86_64:
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: install-prerequisites
-      run: sudo apt-get update && sudo apt-get install -y mpich libmpich-dev
-    - name: build
-      run: ./build all-test mpi-linux-x86_64 smp --build-shared --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C mpi-linux-x86_64-smp/tmp test TESTOPTS="+setcpuaffinity"
-    # Disabled due to several hangs and segfaults.
-    #- name: testp
-    #  run: make -C mpi-linux-x86_64-smp/tmp testp P=4 TESTOPTS="+setcpuaffinity ++ppn 2"
-
-  mpi-linux-x86_64_syncft:
-    timeout-minutes: 90
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: install-prerequisites
-      run: sudo apt-get update && sudo apt-get install -y mpich libmpich-dev
-    - name: build
-      run: ./build all-test mpi-linux-x86_64 syncft -j2 -g
-    - name: test
-      run: make -C mpi-linux-x86_64-syncft/tmp mpisyncfttest
-
-  multicore-darwin-x86_64_projections:
-    timeout-minutes: 60
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: build
-      run: ./build LIBS multicore-darwin-x86_64 -g -j3 --with-production --enable-tracing
-    - name: test
-      run: |
-        make -C multicore-darwin-x86_64/tmp all-test -j3 OPTS="-g -tracemode projections"
-        make -C multicore-darwin-x86_64/tmp test
-    - name: projections
-      run: |
-        git clone https://github.com/UIUC-PPL/projections
-        cd projections
-        make
-        proj=$PWD/bin/projections
-        cd ..
-        files=$(find . -name *.sts)
-        for f in $files; do echo $f; pushd .; cd $(dirname $f); $proj --exit $(basename $f); popd; done
-
-  namd_netlrts-linux-x86_64:
-    # Since NAMD requires a secret to download its source from Gitlab, CI builds from outside PPL
-    # always fail in this test since the secret is not available.
-    env:
-      namd_secret: ${{ secrets.NAMD_CI_USERNAME_TOKEN }}
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: install-prerequisites
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install sfftw-dev
-        # test build without Fortran support
-        sudo apt-get -y remove gfortran
-    - name: build-charm++
-      if: ${{ env.namd_secret != '' }}
-      run: ./build LIBS netlrts-linux-x86_64 -g -j4 --with-production
-    - name: build-namd
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        cd ..
-        git clone https://${{ secrets.NAMD_CI_USERNAME_TOKEN }}@gitlab.com/tcbgUIUC/namd.git
-        cd namd
-        ./config Linux-x86_64-g++ --charm-base ../charm --charm-arch netlrts-linux-x86_64 --without-tcl
-        cd Linux-x86_64-g++
-        make -j4
-    - name: test-namd-alanin
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        cd ../namd/Linux-x86_64-g++
-        ./charmrun ++local +p2 ./namd2 src/alanin
-
-    - name: Cache apoa1 files
-      if: ${{ env.namd_secret != '' }}
-      id: cache-apoa1-files
-      uses: actions/cache@v3
-      with:
-        path: ~/namddata/apoa1
-        key: namd-apoa1-files
-    - name: Download apoa1 files
-      if: steps.cache-apoa1-files.outputs.cache-hit != 'true'
-      run: |
-        mkdir -p ~/namddata
-        cd ~/namddata
-        wget http://www.ks.uiuc.edu/Research/namd/utilities/apoa1.tar.gz
-        tar xzf apoa1.tar.gz
-        sed -i 's,500,250,' apoa1/apoa1.namd # Reduce number of steps
-        sed -i 's,/usr/tmp,/tmp,' apoa1/apoa1.namd # Change output prefix
-        rm apoa1.tar.gz
-
-    - name: test-namd-apoa1
-      if: ${{ env.namd_secret != '' }}
-      run: |
-        cd ../namd/Linux-x86_64-g++
-        ./charmrun ++local +p4 ./namd2 ~/namddata/apoa1/apoa1.namd
-
-  netlrts-darwin-x86_64:
-    timeout-minutes: 60
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: build
-      run: ./build all-test netlrts-darwin-x86_64 --build-shared --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C netlrts-darwin-x86_64/tmp test TESTOPTS="++local"
-    - name: testp P=2
-      run: make -C netlrts-darwin-x86_64/tmp testp P=2 TESTOPTS="++local"
-
-  netlrts-linux-i386:
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: install-prerequisites
-      run: |
-        sudo dpkg --add-architecture i386
-        sudo apt-get update
-        sudo apt-get install -y gcc-multilib g++-multilib gfortran-multilib zlib1g-dev:i386 libjpeg-dev:i386
-    - name: build
-      run: ./build all-test netlrts-linux-i386 --build-shared --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C netlrts-linux-i386/tmp test TESTOPTS="++local"
-    - name: testp P=2
-      run: make -C netlrts-linux-i386/tmp testp P=2 TESTOPTS="++local"
-
-  netlrts-linux-x86_64:
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: build
-      run: ./build all-test netlrts-linux-x86_64 --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C netlrts-linux-x86_64/tmp test TESTOPTS="++local"
-    - name: testp P=2
-      run: make -C netlrts-linux-x86_64/tmp testp P=2 TESTOPTS="++local"
-
-  netlrts-linux-x86_64_smp:
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: build
-      run: ./build all-test netlrts-linux-x86_64 smp --with-production --enable-error-checking --enable-lbuserdata -j3 -g -Werror=vla
-      # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-    - name: test
-      run: make -C netlrts-linux-x86_64-smp/tmp test TESTOPTS="++local +setcpuaffinity +CmiSleepOnIdle"
-    - name: testp P=4 PPN=2
-      run: make -C netlrts-linux-x86_64-smp/tmp testp P=4 TESTOPTS="++local +setcpuaffinity +CmiSleepOnIdle ++ppn 2"
-
-  netlrts-linux-x86_64_syncft:
-    timeout-minutes: 90
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: build
-      run: ./build all-test netlrts-linux-x86_64 syncft -j2 -g
-    - name: test
-      run: make -C netlrts-linux-x86_64-syncft/tmp syncfttest TESTOPTS="++local"
-
-  shellcheck:
-    # See https://github.com/koalaman/shellcheck/wiki
-    # for explanations of ShellCheck error codes
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: ShellCheck
-      run: |
-        # TODO: add more scripts
-        shellcheck \
-          build \
-          buildcmake \
-          package-tarball.sh \
-          relink.script \
-          src/scripts/commitid.sh \
-          src/scripts/testrun \
-          src/arch/multicore/charmrun \
-          src/arch/mpi-win-x86_64/charmrun \
-
-  spack:
-    name: spack_${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Install Spack
-      run: |
-        cd ..
-        git clone --depth 1 https://github.com/spack/spack
-        cd spack
-        [[ $(uname) == "Linux" ]] && sudo apt-get install -y gfortran gfortran-13
-
-        # Use this branch for testing, not main
-        sed -i -e 's,="main",="${{ github.head_ref }}",' var/spack/repos/builtin/packages/charmpp/package.py
-
-        # Use a fork, not the main repo
-        # If in merge_group mode, the branch should be created in the main repo
-        if [[ ${{ github.event_name }} == 'pull_request' ]]; then
-          sed -i -e 's,UIUC-PPL/charm.git,${{github.event.pull_request.head.repo.full_name}},' var/spack/repos/builtin/packages/charmpp/package.py
-        fi
-
-        # Compile with debug symbols
-        sed -i -e 's,build(target,options.append("-g"); build(target,' var/spack/repos/builtin/packages/charmpp/package.py
-
-        # Add +setcpuaffinity option to TESTOPTS
-        sed -i -e 's,\+\+local,++local +setcpuaffinity,' var/spack/repos/builtin/packages/charmpp/package.py
-
-        # No need for automake/autoconf
-        sed -i -e '/automake/d' var/spack/repos/builtin/packages/charmpp/package.py
-        sed -i -e '/autoconf/d' var/spack/repos/builtin/packages/charmpp/package.py
-
-    - name: Build Charm++
-      run: |
-        cd ../spack
-        source share/spack/setup-env.sh
-        spack compiler find
-        spack compiler list
-        spack spec charmpp@main
-        spack install --test root charmpp@main
-
-  ucx-linux-x86_64_openpmix:
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v1
-    # Uncomment the lines below to set up a tmate session for debugging.
-    # See https://github.com/marketplace/actions/debugging-with-tmate for details.
-    # This can't be enabled all the time as the tmate session will wait for a user to connect before running
-    # the build.
-    #- name: Tmate session for debugging
-    #  uses: mxschmitt/action-tmate@v2
-    - name: install-prerequisites
-      run: |
-        sudo apt-get update
-        sudo apt-get install libucx-dev libevent-dev libhwloc-dev
-    - name: build-openpmix
-      run: |
-        wget https://github.com/openpmix/openpmix/releases/download/v3.1.5/pmix-3.1.5.tar.gz
-        tar -xf pmix-3.1.5.tar.gz
-        cd pmix-3.1.5
-        ./configure
-        make -j4
-        sudo make install
-    # OpenMPI is needed to launch Charm++ programs with the UCX backend
-    - name: build-ompi
-      run: |
-        wget https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.5.tar.gz
-        tar -xf openmpi-4.0.5.tar.gz
-        cd openmpi-4.0.5
-        ./configure --enable-mca-no-build=btl-uct --with-pmix=/usr/local
-        make -j4
-        sudo make install
-    - name: build
-      run: ./build all-test ucx-linux-x86_64 openpmix -g -j4 --with-production
-    - name: test
-      run: |
-        export PATH=/usr/local/bin:$PATH
-        export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
-        export OMPI_MCA_rmaps_base_oversubscribe=1
-        make -C ucx-linux-x86_64-openpmix/tests test
-        make -C ucx-linux-x86_64-openpmix/examples test
-
-  verbs-linux-x86_64_smp:
-    # Buildonly test, as the machine layer needs an actual device to run.
-    # Also test packaging a tarball and building from it.
-    # Since it is buildonly, test both build and buildold with the tarball.
-    timeout-minutes: 45
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: install-prerequisites
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install libibverbs-dev
-    - name: package-tarball
-      run: |
-        ./package-tarball.sh
-        cd ..
-        tar -xzf charm/charm-*.tar.gz
-    - name: build-smp
-      run: |
-        cd ../charm-*
-        # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-        ./build all-test verbs-linux-x86_64 smp -j4 -g --build-shared
-    - name: buildold-nonsmp
-      run: |
-        cd ../charm-*
-        # TODO: this should build tests with "-charm-shared". See #2735 on why this is not done currently.
-        ./buildold all-test verbs-linux-x86_64 -j4 -g --build-shared
-
-  netlrts-win-x86_64_mingw64:
-    timeout-minutes: 90
-
-    runs-on: windows-latest
-    defaults:
-      run:
-        shell: msys2 {0}
-
-    steps:
-    - uses: actions/checkout@v4
-    - uses: msys2/setup-msys2@v2
-      with:
-        msystem: MINGW64
-        update: true
-        install: git mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-cmake make libtool autoconf automake
-    - name: build
-      run: |
-        . src/arch/win/vsenv.sh
-        ./build AMPI netlrts-win-x86_64 --with-production -j2
-        make -C netlrts-win-x86_64/tests -j2
-    - name: test
-      run: |
-        make -C netlrts-win-x86_64/tests test TESTOPTS="++local"
-

From 084cf337463ae456cb11fda9e6e3f570d8c515c5 Mon Sep 17 00:00:00 2001
From: Matthias Diener <mdiener@illinois.edu>
Date: Wed, 22 Nov 2023 17:29:35 -0600
Subject: [PATCH 31/31] cleanups, first attempt at pages upload

---
 .github/workflows/autobuild.yml         | 66 ++++++++++++++++++++-----
 .github/workflows/delta-sbatch-slurm.sh | 21 +++++---
 .github/workflows/jobmonitor.sh         | 21 +++++---
 3 files changed, 85 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml
index 93a2e10a27..358a2a3278 100644
--- a/.github/workflows/autobuild.yml
+++ b/.github/workflows/autobuild.yml
@@ -6,9 +6,10 @@ on:
     - cron: '15 06 * * *' # UTC 6:15am, corresponds to 00:15 CST or 01:15 CDT
   push:
     paths:
-      # Also run the build when this file gets modified as part of a PR
+      # Also run the build when these files are modified as part of a PR
       - '.github/workflows/autobuild.yml'
       - '.github/workflows/delta-sbatch-slurm.sh'
+      - '.github/workflows/jobmonitor.sh'
 
 
 # Cancel in progress CI runs when a new run targeting the same PR or branch/tag is triggered.
@@ -18,14 +19,20 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
-  Delta:
+  delta:
     timeout-minutes: 60
 
-    runs-on: delta
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # added or changed files to the repository.
+      contents: write
+
+    # runs-on: delta
+    runs-on: ubuntu-latest
     name: Delta mpi-linux-x86_64  # Could test various builds (e.g., MPI, UCX, ...) via a build matrix
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Host info
       run: |
         set -x
@@ -36,15 +43,52 @@ jobs:
         pwd
     - name: build
       run: |
-        set -ex
-        export target="mpi-linux-x86_64"
-        .github/workflows/jobmonitor.sh .github/workflows/delta-sbatch-slurm.sh
+        set -x
+        # export target="mpi-linux-x86_64"
+        # .github/workflows/jobmonitor.sh .github/workflows/delta-sbatch-slurm.sh
+
+        echo 0 > result.latest
+        echo "hello world" > output.latest
+
+    - name: Check out autobuild-logs repo
+      uses: actions/checkout@v4
+      with:
+        repository: UIUC-PPL/autobuild-logs
+        token: ${{ secrets.GITHUB_TOKEN }}
+        path: autobuild-logs
+
     - name: results
       run: |
+        set -x
+        cat result.latest
         if grep '0' result.latest
         then
-          echo "Success"
+          res="success"
         else
-          echo "Failure"
-        fi	
-        # should also https://github.com/marketplace/actions/send-email
\ No newline at end of file
+          res="failure"
+        fi
+
+        pwd
+        echo $res
+
+        cd autobuild-logs
+        mkdir -p delta/mpi-linux-x86_64
+        cd delta/mpi-linux-x86_64
+
+        cp ../../../output.latest Delta_mpi-linux-x86_64_$(date '+%Y-%m-%dT%H-%M-%S%z')_output_$res.txt
+
+    - uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: Autobuild results for delta/mpi-linux-x86_64
+        repository: autobuild-logs
+
+        # git config --global user.email "autobuild-logs@charm"
+        # git config --global user.name "Autobuild-logs user"
+
+        # git add .
+
+        # git commit -m "Autobuild results for delta/mpi-linux-x86_64"
+
+        # git push
+
+        # # should also https://github.com/marketplace/actions/send-email
diff --git a/.github/workflows/delta-sbatch-slurm.sh b/.github/workflows/delta-sbatch-slurm.sh
index af954d456e..c2e8974f83 100755
--- a/.github/workflows/delta-sbatch-slurm.sh
+++ b/.github/workflows/delta-sbatch-slurm.sh
@@ -7,15 +7,24 @@
 #SBATCH -J autobuild
 #SBATCH -p cpu
 #SBATCH -A mzu-delta-cpu
-#cd $indir
+
 set -x
+
 module load libfabric; module load cmake
-./build all-test $target --with-production --enable-error-checking -j64 -g
-#
+
+mkdir -p $target
+
+# ./build all-test $target --with-production --enable-error-checking -j64 -g
+
 cd $target
-make -C tests test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
-make -C examples test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
-make -C benchmarks test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+# make -C tests test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+# make -C examples test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+# make -C benchmarks test OPTS="$flags" TESTOPTS="$testopts" $maketestopts
+
+# For debugging:
+echo "Just 4 debugging"
+true
+
 # Save make exit status
 status=$?
 echo $status > ../$SLURM_JOBID.result
diff --git a/.github/workflows/jobmonitor.sh b/.github/workflows/jobmonitor.sh
index f4604a8cae..ce618061f0 100755
--- a/.github/workflows/jobmonitor.sh
+++ b/.github/workflows/jobmonitor.sh
@@ -10,9 +10,11 @@ End() {
         $queue_kill $jobid
         exit $1
 }
+
 echo "Submitting batch job for>  $target OPTS=$flags"
 echo " using the command> $queue_qsub $script"
 chmod 755 $script
+
 while [ -z "$jobid" ]
 do
     $queue_qsub $script > .status.$$ 2>&1
@@ -25,12 +27,16 @@ do
     jobid=`cat .status.$$ | tail -1 | awk '{print $4}'`
     rm -f .status.$$
 done
+
 echo "Job enqueued under job ID $jobid"
+
 export output=$jobid.output
 export result=$jobid.result
+
 # kill job if interrupted
 trap 'End 1' 2 3
 retry=0
+
 # Wait for the job to complete, by checking its status
 while [ true ]
 do
@@ -40,9 +46,9 @@ do
     #if [[ $exitstatus != 0 || $linecount != 2 ]]
     if [[ $linecount != 2 ]]
     then
-	# The job is done-- print its output
+        # The job is done-- print its output
         rm tmp.$$
-	# When job hangs, result file does not exist
+        # When job hangs, result file does not exist
         test -f $result && status=`cat $result` || status=1
         echo "==================================== OUTPUT (STDOUT & STDERR) ========================================"
         cat $output
@@ -57,12 +63,15 @@ do
             cat $result
             echo "======================================================================================================"
         fi
-	# mv result and output to result.latest
-	mv $result result.latest
-	mv $output output.latest
+
+        # mv result and output to result.latest
+        mv $result result.latest
+        mv $output output.latest
+
         exit $status
     fi
-# The job is still queued or running-- print status and wait
+
+    # The job is still queued or running-- print status and wait
     tail -1 tmp.$$
     rm tmp.$$
     sleep 60