From d5b7cd0aea017f168a3b32e457e049ddf1124df7 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Wed, 21 Feb 2024 23:09:03 -0700
Subject: [PATCH 01/14] ci: disable power9 and add roci jobs

---
 .gitlab-ci.yml | 152 +++++++++++++++++++++++++++----------------------
 1 file changed, 83 insertions(+), 69 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 017b7bccd6d..24b3030d1b5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,34 +14,49 @@ variables:
   SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
   COLOR_CYAN: "\e[1;36m"
   COLOR_PLAIN: "\e[0m"
+  CLUSTER: darwin
   # set to different spack upstream for testing
   SINGULARITY_EOS_TEMP_SPACK_DIR: ""
   # uncomment for debugging the CI and persist the Spack installation
   # SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/spack-local"
   # DEVEL_SPACK_CHECKOUT: "${CI_PROJECT_DIR}/spack"
 
-before_script:
-  - echo "Running on $(hostname)"
-  - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
-  - export PYTHONNOUSERSITE=1
-  - export SPACK_DISABLE_LOCAL_CONFIG=true
-  - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
-  - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
-  - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
-  - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
-  - |
-    rsync -rpl \
-        --exclude=".git" \
-        --exclude="etc/spack/repos.yaml" \
-        --exclude="etc/spack/packages.yaml" \
-        --include="etc/spack/**" \
-        --include="lib/spack/**" \
-        --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
-        ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
-  - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
-  - spack --version
-  - spack compiler list
-  - section end spack_init
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_TAG
+
+default:
+  tags:
+    - darwin-slurm-shared
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://asc-git.lanl.gov
+  before_script:
+    - echo "Running on $(hostname)"
+    - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
+    - export PYTHONNOUSERSITE=1
+    - export SPACK_DISABLE_LOCAL_CONFIG=true
+    - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
+    - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
+    - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
+    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
+    - |
+      rsync -rpl \
+          --exclude=".git" \
+          --exclude="etc/spack/repos.yaml" \
+          --exclude="etc/spack/packages.yaml" \
+          --include="etc/spack/**" \
+          --include="lib/spack/**" \
+          --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
+          ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
+    - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
+    - spack --version
+    - spack compiler list
+    - section end spack_init
 
 ##########################
 # Build Matrix Variables #
@@ -66,36 +81,27 @@ before_script:
 .nofort: &nofort
   SINGULARITY_USE_FORTRAN: "false"
 
-.skylake: &skylake
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-
-.power9: &power9
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=power9-rhel7"
-
-.x86volta: &x86volta
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"
-
 .python: &python
     SINGULARITY_BUILD_PYTHON: "ON"
 
-#################
-# General Setup #
-#################
+######################
+# Platform Variables #
+######################
 
-.job: &job_def
-  id_tokens:
-    SITE_ID_TOKEN:
-      aud: https://asc-git.lanl.gov
+.rocinante_batch:
   tags:
-    - darwin-slurm-shared
+    - rocinante
+    - batch
   variables:
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-  only:
-    refs:
-      - main
-      - merge_requests
-  except:
-    refs:
+    CLUSTER: rocinante
+    SINGULARITY_EOS_GCC_VERSION: "12.2.0"
+    SCHEDULER_PARAMETERS: "-N 1 -t 30 -A asc-ci --partition=ci --qos=ci --reservation=ci"
+
+.skylake: &skylake
+  SCHEDULER_PARAMETERS: "--nodes=1 -p skylake-gold,skylake-platinum"
+
+.x86volta: &x86volta
+    SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"
 
 #######################
 # Job Script Template #
@@ -103,11 +109,15 @@ before_script:
 
 .test:
   stage: build_n_test
-  extends:
-    - .job
   script:
-    - module load gcc/${SINGULARITY_EOS_GCC_VERSION}
-    - module load cuda/${SINGULARITY_EOS_CUDA_VERSION}
+    - |
+      if [[ "${CLUSTER}" = "rocinante" ]]; then
+        module load PrgEnv-gnu
+        module load gcc/${SINGULARITY_EOS_GCC_VERSION}
+      else
+        module load gcc/${SINGULARITY_EOS_GCC_VERSION}
+        module load cuda/${SINGULARITY_EOS_CUDA_VERSION}
+      fi
     - |
       if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]];
       then
@@ -182,29 +192,12 @@ before_script:
 ########
 
 test_gnu_skylake:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
     <<: *python
 
-test_gnu_power9:
-  <<: *job_def
-  extends: .test
-  variables:
-    <<: *power9
-    <<: *python
-
-test_gnu_power9_gpu:
-  <<: *job_def
-  extends: .test
-  variables:    
-    <<: *power9
-    <<: *buildtype_release
-    <<: *gpu
-
 test_x86_volta_gpu:
-  <<: *job_def
   extends: .test
   variables:    
     <<: *x86volta
@@ -212,7 +205,6 @@ test_x86_volta_gpu:
     <<: *gpu
 
 install_gnu_skylake_fort:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
@@ -220,9 +212,31 @@ install_gnu_skylake_fort:
     <<: *fort
 
 install_gnu_skylake_nofort:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
     <<: *python
     <<: *nofort
+
+test_gnu_rocinante:
+  extends:
+   - .rocinante_batch
+   - .test
+  variables:
+    <<: *python
+
+install_gnu_rocinante_fort:
+  extends:
+   - .rocinante_batch
+   - .test
+  variables:
+    <<: *python
+    <<: *fort
+
+install_gnu_rocinante_nofort:
+  extends:
+   - .rocinante_batch
+   - .test
+  variables:
+    <<: *python
+    <<: *nofort

From bf92fef95b07164913fa2e2d88d83c1d3e3a5652 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Thu, 22 Feb 2024 00:06:22 -0700
Subject: [PATCH 02/14] Correct test/test_eos_tabulated.cpp

---
 test/test_eos_tabulated.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/test/test_eos_tabulated.cpp b/test/test_eos_tabulated.cpp
index 3944b37ce15..9844677d87d 100644
--- a/test/test_eos_tabulated.cpp
+++ b/test/test_eos_tabulated.cpp
@@ -23,10 +23,6 @@
 #include <ports-of-call/portable_errors.hpp>
 #include <singularity-eos/eos/eos.hpp>
 
-#ifdef SINGULARITY_BUILD_CLOSURE
-#include <singularity-eos/eos/singularity_eos.hpp>
-#endif
-
 #ifndef CATCH_CONFIG_FAST_COMPILE
 #define CATCH_CONFIG_FAST_COMPILE
 #include <catch2/catch_test_macros.hpp>

From fad8a87851cd143aa348bcbeba4c3d16d875e9a3 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Thu, 22 Feb 2024 00:35:42 -0700
Subject: [PATCH 03/14] cmake: build singularity_eos.cpp even without Fortran

---
 singularity-eos/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/singularity-eos/CMakeLists.txt b/singularity-eos/CMakeLists.txt
index ae5dee1bed0..5450cb43e41 100644
--- a/singularity-eos/CMakeLists.txt
+++ b/singularity-eos/CMakeLists.txt
@@ -59,6 +59,8 @@ register_headers(
 
 if (SINGULARITY_BUILD_CLOSURE)
   register_headers(closure/mixed_cell_models.hpp)
+  register_headers(eos/singularity_eos.hpp)
+  register_srcs(eos/singularity_eos.cpp)
   if (SINGULARITY_USE_FORTRAN)
     register_srcs(eos/get_sg_eos.cpp)
     if (SINGULARITY_USE_KOKKOS)
@@ -77,7 +79,5 @@ endif()
 if (SINGULARITY_USE_FORTRAN)
   register_srcs(
     eos/singularity_eos.f90
-    eos/singularity_eos.cpp
   )
-  register_headers(eos/singularity_eos.hpp)
 endif()

From c543f2e3ecf5e406cbdd9a09f36915984b49c88f Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Fri, 23 Feb 2024 14:41:30 -0700
Subject: [PATCH 04/14] ci: bump GCC version to 10.3.0

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 24b3030d1b5..9955fb45688 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ stages:
 
 variables:
   GIT_SUBMODULE_STRATEGY: normal
-  SINGULARITY_EOS_GCC_VERSION: "9.3.0"
+  SINGULARITY_EOS_GCC_VERSION: "10.3.0"
   SINGULARITY_EOS_OPENMPI_VERSION: "4.1.3"
   SINGULARITY_EOS_CUDA_VERSION: "11.6.0"
   SINGULARITY_EOS_BUILD_TYPE: "RelWithDebInfo"

From 98f67eae81f48e23a09a0f87ac937ae73d83f8e8 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Fri, 23 Feb 2024 15:00:05 -0700
Subject: [PATCH 05/14] ci: use cray-mpich on roci

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9955fb45688..d844c910b7b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -95,6 +95,7 @@ default:
   variables:
     CLUSTER: rocinante
     SINGULARITY_EOS_GCC_VERSION: "12.2.0"
+    SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cray-mpich"
     SCHEDULER_PARAMETERS: "-N 1 -t 30 -A asc-ci --partition=ci --qos=ci --reservation=ci"
 
 .skylake: &skylake

From 3bfd2a0b49c4d7baecff06a3758c3e925e201fef Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Fri, 23 Feb 2024 17:25:07 -0700
Subject: [PATCH 06/14] ci: force kokkos to use std=14

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d844c910b7b..b5508563825 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -73,7 +73,7 @@ default:
 
 .gpu: &gpu
   SINGULARITY_USE_CUDA: "true"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+hdf5 cuda_arch=70 +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+hdf5 cuda_arch=70 +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
 
 .fort: &fort
   SINGULARITY_USE_FORTRAN: "true"

From 2d70df604fd9ca0ebd4b24b68e2254f2e47e336d Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Wed, 28 Feb 2024 08:48:07 -0700
Subject: [PATCH 07/14] ci: add a100 jobs

---
 .gitlab-ci.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b5508563825..07f2dec3939 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -73,7 +73,7 @@ default:
 
 .gpu: &gpu
   SINGULARITY_USE_CUDA: "true"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+hdf5 cuda_arch=70 +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+hdf5 cuda_arch=${SINGULARITY_CUDA_ARCH} +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
 
 .fort: &fort
   SINGULARITY_USE_FORTRAN: "true"
@@ -102,8 +102,13 @@ default:
   SCHEDULER_PARAMETERS: "--nodes=1 -p skylake-gold,skylake-platinum"
 
 .x86volta: &x86volta
+    SINGULARITY_CUDA_ARCH: 70
     SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"
 
+.a100: &a100
+    SINGULARITY_CUDA_ARCH: 80
+    SCHEDULER_PARAMETERS: "--nodes=1 --partition=shared-gpu-ampere"
+
 #######################
 # Job Script Template #
 #######################
@@ -200,11 +205,18 @@ test_gnu_skylake:
 
 test_x86_volta_gpu:
   extends: .test
-  variables:    
+  variables:
     <<: *x86volta
     <<: *buildtype_release
     <<: *gpu
 
+test_x86_ampere_gpu:
+  extends: .test
+  variables:
+    <<: *a100
+    <<: *buildtype_release
+    <<: *gpu
+
 install_gnu_skylake_fort:
   extends: .test
   variables:

From 14b2f3ff0ef6ce207c57acaa7311f8ff66670209 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Wed, 6 Mar 2024 13:50:07 -0700
Subject: [PATCH 08/14] ci: report spack upstream that is used

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 07f2dec3939..3500d21de37 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -44,6 +44,7 @@ default:
     - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
     - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
     - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
+    - echo "Upstream ${SINGULARITY_EOS_SPACK_DIR}"
     - |
       rsync -rpl \
           --exclude=".git" \

From 92378d92afc16adfc1025c5c30c3e0ee3a6f0a26 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Wed, 6 Mar 2024 14:47:00 -0700
Subject: [PATCH 09/14] Update spackage

---
 spack-repo/packages/singularity-eos/package.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/spack-repo/packages/singularity-eos/package.py b/spack-repo/packages/singularity-eos/package.py
index 5d5c58617e7..2f3dd0e4f2c 100644
--- a/spack-repo/packages/singularity-eos/package.py
+++ b/spack-repo/packages/singularity-eos/package.py
@@ -144,7 +144,6 @@ class SingularityEos(CMakePackage, CudaPackage):
     for _flag in ("~cuda", "+cuda", "~openmp", "+openmp"):
         depends_on("kokkos ~shared" + _flag, when="+kokkos" + _flag)
         depends_on("kokkos-kernels" + _flag, when="+kokkos-kernels" + _flag)
-        depends_on("spiner" + _flag, when="+kokkos" + _flag)
 
     # specfic specs when using GPU/cuda offloading
     # TODO remove +wrapper for clang builds
@@ -158,7 +157,6 @@ class SingularityEos(CMakePackage, CudaPackage):
     for _flag in list(CudaPackage.cuda_arch_values):
         depends_on("kokkos cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag)
         depends_on("kokkos-kernels cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag)
-        depends_on("spiner cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag)
 
     conflicts("cuda_arch=none", when="+cuda", msg="CUDA architecture is required")
 

From 3ca5654b3614a15da0faf73d169ef8a1435fb573 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Thu, 7 Mar 2024 11:03:39 -0700
Subject: [PATCH 10/14] Add xcap_deployment repo

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3500d21de37..2ae74770e78 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -132,6 +132,7 @@ default:
         mkdir -p spack_build_env
         spack env create spack_build_env
         spack env activate spack_build_env
+        spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
         spack repo add spack-repo
         spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
         spack dev-build -j $(nproc) ${SINGULARITY_EOS_SPACK_SPEC}
@@ -142,6 +143,7 @@ default:
     - mkdir -p spack_env
     - spack env create spack_env
     - spack env activate spack_env
+    - spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
     - spack repo add spack-repo
     - spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
     - spack spec -I ${SINGULARITY_EOS_SPACK_SPEC}

From 8c468bf06e6615c8a76c0ac1b7769951ab1e8f61 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Thu, 7 Mar 2024 11:35:40 -0700
Subject: [PATCH 11/14] Use submodules for offline nodes

---
 .gitlab-ci.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2ae74770e78..78ef77bb424 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -134,6 +134,9 @@ default:
         spack env activate spack_build_env
         spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
         spack repo add spack-repo
+        spack develop -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
+        spack develop -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
+        spack develop -p ${CI_PROJECT_DIR} singularity-eos@main
         spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
         spack dev-build -j $(nproc) ${SINGULARITY_EOS_SPACK_SPEC}
         spack env deactivate
@@ -145,6 +148,9 @@ default:
     - spack env activate spack_env
     - spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
     - spack repo add spack-repo
+    - spack develop -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
+    - spack develop -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
+    - spack develop -p ${CI_PROJECT_DIR} singularity-eos@main
     - spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
     - spack spec -I ${SINGULARITY_EOS_SPACK_SPEC}
     - spack add ${SINGULARITY_EOS_SPACK_SPEC}

From 507aa186e9fc8eb1a1d21ea0a26149ca0934b672 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Thu, 7 Mar 2024 12:10:14 -0700
Subject: [PATCH 12/14] Update spiner and ports-of-call submodule commit

---
 utils/ports-of-call | 2 +-
 utils/spiner        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/utils/ports-of-call b/utils/ports-of-call
index 1ae17ad8b6d..58ce1181b2d 160000
--- a/utils/ports-of-call
+++ b/utils/ports-of-call
@@ -1 +1 @@
-Subproject commit 1ae17ad8b6d51b14e46a13d329a09f76c7e8aa29
+Subproject commit 58ce1181b2d835bd32673ad70550c9130381f91b
diff --git a/utils/spiner b/utils/spiner
index c76462824ca..cc089ccb3a1 160000
--- a/utils/spiner
+++ b/utils/spiner
@@ -1 +1 @@
-Subproject commit c76462824ca56e83ce6257f05da3454af042ac0c
+Subproject commit cc089ccb3a1f57afdc21becaaab286217589cf7c

From f03ea428a210e0a79d88521eb33277837171d27a Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Thu, 7 Mar 2024 12:12:11 -0700
Subject: [PATCH 13/14] ci: add missing +spiner

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 78ef77bb424..013b8e8262c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,7 +11,7 @@ variables:
   SINGULARITY_EOS_OPENMPI_VERSION: "4.1.3"
   SINGULARITY_EOS_CUDA_VERSION: "11.6.0"
   SINGULARITY_EOS_BUILD_TYPE: "RelWithDebInfo"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
   COLOR_CYAN: "\e[1;36m"
   COLOR_PLAIN: "\e[0m"
   CLUSTER: darwin
@@ -74,7 +74,7 @@ default:
 
 .gpu: &gpu
   SINGULARITY_USE_CUDA: "true"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+hdf5 cuda_arch=${SINGULARITY_CUDA_ARCH} +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+spiner+hdf5 cuda_arch=${SINGULARITY_CUDA_ARCH} +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
 
 .fort: &fort
   SINGULARITY_USE_FORTRAN: "true"
@@ -96,7 +96,7 @@ default:
   variables:
     CLUSTER: rocinante
     SINGULARITY_EOS_GCC_VERSION: "12.2.0"
-    SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cray-mpich"
+    SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cray-mpich"
     SCHEDULER_PARAMETERS: "-N 1 -t 30 -A asc-ci --partition=ci --qos=ci --reservation=ci"
 
 .skylake: &skylake

From f75632b64b86b7a00c904348e8b379eb2aa93b91 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Fri, 8 Mar 2024 09:00:34 -0700
Subject: [PATCH 14/14] ci: prepare goldfiles.tar.gz prior to other jobs

This allows offline jobs to continue.
---
 .gitlab-ci.yml      | 28 ++++++++++++++++++++++++++++
 test/CMakeLists.txt |  4 +++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 013b8e8262c..4c171cd88a0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,10 +3,12 @@
 ###################
 
 stages:
+  - prepare
   - build_n_test
 
 variables:
   GIT_SUBMODULE_STRATEGY: normal
+  SINGULARITY_GOLDFILES_VERSION: "goldfiles-1.8.0"
   SINGULARITY_EOS_GCC_VERSION: "10.3.0"
   SINGULARITY_EOS_OPENMPI_VERSION: "4.1.3"
   SINGULARITY_EOS_CUDA_VERSION: "11.6.0"
@@ -89,6 +91,11 @@ default:
 # Platform Variables #
 ######################
 
+.rocinante_shell:
+  tags:
+    - rocinante
+    - shell
+
 .rocinante_batch:
   tags:
     - rocinante
@@ -114,6 +121,14 @@ default:
 # Job Script Template #
 #######################
 
+.download_prereq:
+  stage: prepare
+  script:
+   - wget https://github.com/lanl/singularity-eos/releases/download/${SINGULARITY_GOLDFILES_VERSION}/goldfiles.tar.gz
+  artifacts:
+    paths:
+      - goldfiles.tar.gz
+
 .test:
   stage: build_n_test
   script:
@@ -183,6 +198,7 @@ default:
             -DSINGULARITY_USE_KOKKOS=${SINGULARITY_USE_CUDA:-OFF} \
             -DSINGULARITY_USE_KOKKOSKERNELS=${SINGULARITY_USE_CUDA:-OFF} \
             -DSINGULARITY_USE_FORTRAN=${SINGULARITY_USE_FORTRAN:-OFF} \
+            -DSINGULARITY_GOLDFILE_URL=file://${CI_PROJECT_DIR}/goldfiles.tar.gz \
             ..
     - make -j $(nproc) || make VERBOSE=1
     - |
@@ -206,13 +222,19 @@ default:
 # Jobs #
 ########
 
+download_prereq:
+  extends:
+    - .download_prereq
+
 test_gnu_skylake:
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *skylake
     <<: *python
 
 test_x86_volta_gpu:
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *x86volta
@@ -220,6 +242,7 @@ test_x86_volta_gpu:
     <<: *gpu
 
 test_x86_ampere_gpu:
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *a100
@@ -227,6 +250,7 @@ test_x86_ampere_gpu:
     <<: *gpu
 
 install_gnu_skylake_fort:
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *skylake
@@ -234,6 +258,7 @@ install_gnu_skylake_fort:
     <<: *fort
 
 install_gnu_skylake_nofort:
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *skylake
@@ -241,6 +266,7 @@ install_gnu_skylake_nofort:
     <<: *nofort
 
 test_gnu_rocinante:
+  needs: [download_prereq]
   extends:
    - .rocinante_batch
    - .test
@@ -248,6 +274,7 @@ test_gnu_rocinante:
     <<: *python
 
 install_gnu_rocinante_fort:
+  needs: [download_prereq]
   extends:
    - .rocinante_batch
    - .test
@@ -256,6 +283,7 @@ install_gnu_rocinante_fort:
     <<: *fort
 
 install_gnu_rocinante_nofort:
+  needs: [download_prereq]
   extends:
    - .rocinante_batch
    - .test
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 3c997fc51ca..7169015d351 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -110,8 +110,10 @@ endif()
 
 if(SINGULARITY_TEST_STELLAR_COLLAPSE)
   set(SINGULARITY_GOLDFILE_URL
-      https://github.com/lanl/singularity-eos/releases/download/${SINGULARITY_GOLDFILES_VERSION}/goldfiles.tar.gz
+      "https://github.com/lanl/singularity-eos/releases/download/${SINGULARITY_GOLDFILES_VERSION}/goldfiles.tar.gz"
+      CACHE STRING "URL to goldfiles.tar.gz"
   )
+  mark_as_advanced(SINGULARITY_GOLDFILE_URL)
   message(
     STATUS
       "Attempting to download gold files for regression tests. "