ci: disable power9, add a100 and roci jobs

.gitlab-ci.yml

@@ -3,45 +3,63 @@
 ###################
 
 stages:
+  - prepare
   - build_n_test
 
 variables:
   GIT_SUBMODULE_STRATEGY: normal
-  SINGULARITY_EOS_GCC_VERSION: "9.3.0"
+  SINGULARITY_GOLDFILES_VERSION: "goldfiles-1.8.0"
+  SINGULARITY_EOS_GCC_VERSION: "10.3.0"
   SINGULARITY_EOS_OPENMPI_VERSION: "4.1.3"
   SINGULARITY_EOS_CUDA_VERSION: "11.6.0"
   SINGULARITY_EOS_BUILD_TYPE: "RelWithDebInfo"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
   COLOR_CYAN: "\e[1;36m"
   COLOR_PLAIN: "\e[0m"
+  CLUSTER: darwin
   # set to different spack upstream for testing
   SINGULARITY_EOS_TEMP_SPACK_DIR: ""
   # uncomment for debugging the CI and persist the Spack installation
   # SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/spack-local"
   # DEVEL_SPACK_CHECKOUT: "${CI_PROJECT_DIR}/spack"
 
-before_script:
-  - echo "Running on $(hostname)"
-  - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
-  - export PYTHONNOUSERSITE=1
-  - export SPACK_DISABLE_LOCAL_CONFIG=true
-  - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
-  - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
-  - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
-  - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
-  - |
-    rsync -rpl \
-        --exclude=".git" \
-        --exclude="etc/spack/repos.yaml" \
-        --exclude="etc/spack/packages.yaml" \
-        --include="etc/spack/**" \
-        --include="lib/spack/**" \
-        --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
-        ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
-  - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
-  - spack --version
-  - spack compiler list
-  - section end spack_init
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_TAG
+
+default:
+  tags:
+    - darwin-slurm-shared
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://asc-git.lanl.gov
+  before_script:
+    - echo "Running on $(hostname)"
+    - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
+    - export PYTHONNOUSERSITE=1
+    - export SPACK_DISABLE_LOCAL_CONFIG=true
+    - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
+    - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
+    - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
+    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
+    - echo "Upstream ${SINGULARITY_EOS_SPACK_DIR}"
+    - |
+      rsync -rpl \
+          --exclude=".git" \
+          --exclude="etc/spack/repos.yaml" \
+          --exclude="etc/spack/packages.yaml" \
+          --include="etc/spack/**" \
+          --include="lib/spack/**" \
+          --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
+          ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
+    - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
+    - spack --version
+    - spack compiler list
+    - section end spack_init
 
 ##########################
 # Build Matrix Variables #
@@ -58,64 +76,82 @@ before_script:
 
 .gpu: &gpu
   SINGULARITY_USE_CUDA: "true"
-  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+hdf5 cuda_arch=70 +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
+  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+spiner+hdf5 cuda_arch=${SINGULARITY_CUDA_ARCH} +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^kokkos std=14 ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
 
 .fort: &fort
   SINGULARITY_USE_FORTRAN: "true"
 
 .nofort: &nofort
   SINGULARITY_USE_FORTRAN: "false"
 
-.skylake: &skylake
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-
-.power9: &power9
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=power9-rhel7"
-
-.x86volta: &x86volta
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"
-
 .python: &python
     SINGULARITY_BUILD_PYTHON: "ON"
 
-#################
-# General Setup #
-#################
+######################
+# Platform Variables #
+######################
 
-.job: &job_def
-  id_tokens:
-    SITE_ID_TOKEN:
-      aud: https://asc-git.lanl.gov
+.rocinante_shell:
   tags:
-    - darwin-slurm-shared
+    - rocinante
+    - shell
+
+.rocinante_batch:
+  tags:
+    - rocinante
+    - batch
   variables:
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-  only:
-    refs:
-      - main
-      - merge_requests
-  except:
-    refs:
+    CLUSTER: rocinante
+    SINGULARITY_EOS_GCC_VERSION: "12.2.0"
+    SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+spiner+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cray-mpich"
+    SCHEDULER_PARAMETERS: "-N 1 -t 30 -A asc-ci --partition=ci --qos=ci --reservation=ci"
+
+.skylake: &skylake
+  SCHEDULER_PARAMETERS: "--nodes=1 -p skylake-gold,skylake-platinum"
+
+.x86volta: &x86volta
+    SINGULARITY_CUDA_ARCH: 70
+    SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"
+
+.a100: &a100
+    SINGULARITY_CUDA_ARCH: 80
+    SCHEDULER_PARAMETERS: "--nodes=1 --partition=shared-gpu-ampere"
 
 #######################
 # Job Script Template #
 #######################
 
+.download_prereq:
+  stage: prepare
+  script:
+   - wget https://github.com/lanl/singularity-eos/releases/download/${SINGULARITY_GOLDFILES_VERSION}/goldfiles.tar.gz
+  artifacts:
+    paths:
+      - goldfiles.tar.gz
+
 .test:
   stage: build_n_test
-  extends:
-    - .job
   script:
-    - module load gcc/${SINGULARITY_EOS_GCC_VERSION}
-    - module load cuda/${SINGULARITY_EOS_CUDA_VERSION}
+    - |
+      if [[ "${CLUSTER}" = "rocinante" ]]; then
+        module load PrgEnv-gnu
+        module load gcc/${SINGULARITY_EOS_GCC_VERSION}
+      else
+        module load gcc/${SINGULARITY_EOS_GCC_VERSION}
+        module load cuda/${SINGULARITY_EOS_CUDA_VERSION}
+      fi
     - |
       if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]];
       then
         section start "spack_build[collapsed=true]" "Building via Spack"
         mkdir -p spack_build_env
         spack env create spack_build_env
         spack env activate spack_build_env
+        spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
         spack repo add spack-repo
+        spack develop -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
+        spack develop -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
+        spack develop -p ${CI_PROJECT_DIR} singularity-eos@main
         spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
         spack dev-build -j $(nproc) ${SINGULARITY_EOS_SPACK_SPEC}
         spack env deactivate
@@ -125,7 +161,11 @@ before_script:
     - mkdir -p spack_env
     - spack env create spack_env
     - spack env activate spack_env
+    - spack repo add ${DEVEL_SPACK_CHECKOUT}/var/spack/repos/xcap_deployment
     - spack repo add spack-repo
+    - spack develop -p ${CI_PROJECT_DIR}/utils/spiner spiner@main
+    - spack develop -p ${CI_PROJECT_DIR}/utils/ports-of-call ports-of-call@main
+    - spack develop -p ${CI_PROJECT_DIR} singularity-eos@main
     - spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
     - spack spec -I ${SINGULARITY_EOS_SPACK_SPEC}
     - spack add ${SINGULARITY_EOS_SPACK_SPEC}
@@ -158,6 +198,7 @@ before_script:
             -DSINGULARITY_USE_KOKKOS=${SINGULARITY_USE_CUDA:-OFF} \
             -DSINGULARITY_USE_KOKKOSKERNELS=${SINGULARITY_USE_CUDA:-OFF} \
             -DSINGULARITY_USE_FORTRAN=${SINGULARITY_USE_FORTRAN:-OFF} \
+            -DSINGULARITY_GOLDFILE_URL=file://${CI_PROJECT_DIR}/goldfiles.tar.gz \
             ..
     - make -j $(nproc) || make VERBOSE=1
     - |
@@ -181,48 +222,71 @@ before_script:
 # Jobs #
 ########
 
+download_prereq:
+  extends:
+    - .download_prereq
+
 test_gnu_skylake:
-  <<: *job_def
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *skylake
     <<: *python
 
-test_gnu_power9:
-  <<: *job_def
+test_x86_volta_gpu:
+  needs: [download_prereq]
   extends: .test
   variables:
-    <<: *power9
-    <<: *python
-
-test_gnu_power9_gpu:
-  <<: *job_def
-  extends: .test
-  variables:    
-    <<: *power9
+    <<: *x86volta
     <<: *buildtype_release
     <<: *gpu
 
-test_x86_volta_gpu:
-  <<: *job_def
+test_x86_ampere_gpu:
+  needs: [download_prereq]
   extends: .test
-  variables:    
-    <<: *x86volta
+  variables:
+    <<: *a100
     <<: *buildtype_release
     <<: *gpu
 
 install_gnu_skylake_fort:
-  <<: *job_def
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *skylake
     <<: *python
     <<: *fort
 
 install_gnu_skylake_nofort:
-  <<: *job_def
+  needs: [download_prereq]
   extends: .test
   variables:
     <<: *skylake
     <<: *python
     <<: *nofort
+
+test_gnu_rocinante:
+  needs: [download_prereq]
+  extends:
+   - .rocinante_batch
+   - .test
+  variables:
+    <<: *python
+
+install_gnu_rocinante_fort:
+  needs: [download_prereq]
+  extends:
+   - .rocinante_batch
+   - .test
+  variables:
+    <<: *python
+    <<: *fort
+
+install_gnu_rocinante_nofort:
+  needs: [download_prereq]
+  extends:
+   - .rocinante_batch
+   - .test
+  variables:
+    <<: *python
+    <<: *nofort

singularity-eos/CMakeLists.txt

@@ -59,6 +59,8 @@ register_headers(
 
 if (SINGULARITY_BUILD_CLOSURE)
   register_headers(closure/mixed_cell_models.hpp)
+  register_headers(eos/singularity_eos.hpp)
+  register_srcs(eos/singularity_eos.cpp)
   if (SINGULARITY_USE_FORTRAN)
     register_srcs(eos/get_sg_eos.cpp)
     if (SINGULARITY_USE_KOKKOS)
@@ -77,7 +79,5 @@ endif()
 if (SINGULARITY_USE_FORTRAN)
   register_srcs(
     eos/singularity_eos.f90
-    eos/singularity_eos.cpp
   )
-  register_headers(eos/singularity_eos.hpp)
 endif()

spack-repo/packages/singularity-eos/package.py

@@ -144,7 +144,6 @@ class SingularityEos(CMakePackage, CudaPackage):
     for _flag in ("~cuda", "+cuda", "~openmp", "+openmp"):
         depends_on("kokkos ~shared" + _flag, when="+kokkos" + _flag)
         depends_on("kokkos-kernels" + _flag, when="+kokkos-kernels" + _flag)
-        depends_on("spiner" + _flag, when="+kokkos" + _flag)
 
     # specfic specs when using GPU/cuda offloading
     # TODO remove +wrapper for clang builds
@@ -158,7 +157,6 @@ class SingularityEos(CMakePackage, CudaPackage):
     for _flag in list(CudaPackage.cuda_arch_values):
         depends_on("kokkos cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag)
         depends_on("kokkos-kernels cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag)
-        depends_on("spiner cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag)
 
     conflicts("cuda_arch=none", when="+cuda", msg="CUDA architecture is required")
 

test/CMakeLists.txt

@@ -110,8 +110,10 @@ endif()
 
 if(SINGULARITY_TEST_STELLAR_COLLAPSE)
   set(SINGULARITY_GOLDFILE_URL
-      https://github.com/lanl/singularity-eos/releases/download/${SINGULARITY_GOLDFILES_VERSION}/goldfiles.tar.gz
+      "https://github.com/lanl/singularity-eos/releases/download/${SINGULARITY_GOLDFILES_VERSION}/goldfiles.tar.gz"
+      CACHE STRING "URL to goldfiles.tar.gz"
   )
+  mark_as_advanced(SINGULARITY_GOLDFILE_URL)
   message(
     STATUS
       "Attempting to download gold files for regression tests. "

test/test_eos_tabulated.cpp

@@ -23,10 +23,6 @@
 #include <ports-of-call/portable_errors.hpp>
 #include <singularity-eos/eos/eos.hpp>
 
-#ifdef SINGULARITY_BUILD_CLOSURE
-#include <singularity-eos/eos/singularity_eos.hpp>
-#endif
-
 #ifndef CATCH_CONFIG_FAST_COMPILE
 #define CATCH_CONFIG_FAST_COMPILE
 #include <catch2/catch_test_macros.hpp>