From 434a498bb10d67946be4e8979d6a406a20672027 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Wed, 21 Feb 2024 23:09:03 -0700
Subject: [PATCH] ci: disable power9 and add roci jobs

---
 .gitlab-ci.yml | 163 ++++++++++++++++++++++++++++---------------------
 1 file changed, 94 insertions(+), 69 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 017b7bccd6d..3a6277c41f3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,28 +20,42 @@ variables:
   # SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/spack-local"
   # DEVEL_SPACK_CHECKOUT: "${CI_PROJECT_DIR}/spack"
 
-before_script:
-  - echo "Running on $(hostname)"
-  - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
-  - export PYTHONNOUSERSITE=1
-  - export SPACK_DISABLE_LOCAL_CONFIG=true
-  - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
-  - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
-  - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
-  - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
-  - |
-    rsync -rpl \
-        --exclude=".git" \
-        --exclude="etc/spack/repos.yaml" \
-        --exclude="etc/spack/packages.yaml" \
-        --include="etc/spack/**" \
-        --include="lib/spack/**" \
-        --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
-        ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
-  - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
-  - spack --version
-  - spack compiler list
-  - section end spack_init
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_TAG
+
+default:
+  tags:
+    - darwin-slurm-shared
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://asc-git.lanl.gov
+  before_script:
+    - echo "Running on $(hostname)"
+    - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
+    - export PYTHONNOUSERSITE=1
+    - export SPACK_DISABLE_LOCAL_CONFIG=true
+    - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
+    - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
+    - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
+    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
+    - |
+      rsync -rpl \
+          --exclude=".git" \
+          --exclude="etc/spack/repos.yaml" \
+          --exclude="etc/spack/packages.yaml" \
+          --include="etc/spack/**" \
+          --include="lib/spack/**" \
+          --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
+          ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
+    - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
+    - spack --version
+    - spack compiler list
+    - section end spack_init
 
 ##########################
 # Build Matrix Variables #
@@ -66,36 +80,48 @@ before_script:
 .nofort: &nofort
   SINGULARITY_USE_FORTRAN: "false"
 
-.skylake: &skylake
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-
-.power9: &power9
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=power9-rhel7"
-
-.x86volta: &x86volta
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"
-
 .python: &python
     SINGULARITY_BUILD_PYTHON: "ON"
 
-#################
-# General Setup #
-#################
+######################
+# Platform Variables #
+######################
+
+.rocinante_shell: &rocinante_shell
+  tags: &rocinante_shell_tags
+    - rocinante
+    - shell
+    - jacamar
+    - slurm
+
+.rocinante_batch: &rocinante_batch
+  tags: &rocinante_batch_tags
+    - rocinante
+    - batch
+    - jacamar
+    - slurm
+  variables: &rocinante_batch_var
+    SCHEDULER_PARAMETERS: "--nodes=1 --time 1:00:00 --account=asc-ci --qos=ci --reservation=ci --partition=ci"
+
+.darwin_shell: &darwin_shell
+  tags: &darwin_shell_tags
+    - darwin
+    - jacamar
+    - shell
+
+.darwin_batch: &darwin_batch
+  tags: &darwin_batch_tags
+    - darwin
+    - jacamar
+    - batch
+  variables: &darwin_batch_var
+    SCHEDULER_PARAMETERS: "--nodes=1 -p skylake-gold,skylake-platinum"
 
-.job: &job_def
-  id_tokens:
-    SITE_ID_TOKEN:
-      aud: https://asc-git.lanl.gov
-  tags:
-    - darwin-slurm-shared
-  variables:
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-  only:
-    refs:
-      - main
-      - merge_requests
-  except:
-    refs:
+.skylake: &skylake
+  SCHEDULER_PARAMETERS: "--nodes=1 -p skylake-gold,skylake-platinum"
+
+.x86volta: &x86volta
+    SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"
 
 #######################
 # Job Script Template #
@@ -103,8 +129,6 @@ before_script:
 
 .test:
   stage: build_n_test
-  extends:
-    - .job
   script:
     - module load gcc/${SINGULARITY_EOS_GCC_VERSION}
     - module load cuda/${SINGULARITY_EOS_CUDA_VERSION}
@@ -182,29 +206,12 @@ before_script:
 ########
 
 test_gnu_skylake:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
     <<: *python
 
-test_gnu_power9:
-  <<: *job_def
-  extends: .test
-  variables:
-    <<: *power9
-    <<: *python
-
-test_gnu_power9_gpu:
-  <<: *job_def
-  extends: .test
-  variables:    
-    <<: *power9
-    <<: *buildtype_release
-    <<: *gpu
-
 test_x86_volta_gpu:
-  <<: *job_def
   extends: .test
   variables:    
     <<: *x86volta
@@ -212,7 +219,6 @@ test_x86_volta_gpu:
     <<: *gpu
 
 install_gnu_skylake_fort:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
@@ -220,9 +226,28 @@ install_gnu_skylake_fort:
     <<: *fort
 
 install_gnu_skylake_nofort:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
     <<: *python
     <<: *nofort
+
+test_gnu_rocinante:
+  <<: *rocinante_batch
+  extends: .test
+  variables:
+    <<: *python
+
+install_gnu_rocinante_fort:
+  <<: *rocinante_batch
+  extends: .test
+  variables:
+    <<: *python
+    <<: *fort
+
+install_gnu_rocinante_nofort:
+  <<: *rocinante_batch
+  extends: .test
+  variables:
+    <<: *python
+    <<: *nofort