From 047c3662f854221785906a446e6ff50dd5e42fde Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Nov 2023 14:06:19 +0100 Subject: [PATCH 1/6] update script for building for software.eessi.io (version 2023.06) --- ...l-software.sh => EESSI-install-software.sh | 2 +- README.md | 16 ++++++------- bot/build.sh | 16 ++++++------- bot/inspect.sh | 20 ++++++++-------- check_missing_installations.sh | 2 +- create_directory_tarballs.sh | 4 ++-- create_tarball.sh | 24 +++++++++---------- eessi_container.sh | 14 +++++------ init/Magic_Castle/bash | 4 ++-- .../{eessi_pilot_python3 => eessi_python3} | 8 +++---- init/bash | 6 ++--- init/eessi_defaults | 4 ++-- init/eessi_environment_variables | 6 ++--- init/minimal_eessi_env | 4 ++-- install_software_layer.sh | 2 +- run_in_compat_layer_env.sh | 12 +++++----- 16 files changed, 72 insertions(+), 72 deletions(-) rename EESSI-pilot-install-software.sh => EESSI-install-software.sh (98%) rename init/Magic_Castle/{eessi_pilot_python3 => eessi_python3} (73%) diff --git a/EESSI-pilot-install-software.sh b/EESSI-install-software.sh similarity index 98% rename from EESSI-pilot-install-software.sh rename to EESSI-install-software.sh index 9586abde1d..234f56d518 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-install-software.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to install EESSI pilot software stack (version set through init/eessi_defaults) +# Script to install EESSI software stack (version set through init/eessi_defaults) # see example parsing of command line arguments at # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop diff --git a/README.md b/README.md index e24de45b0e..ab73ad7579 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,22 @@ # Software layer -The software layer of the EESSI project uses [EasyBuild](https://easybuild.readthedocs.io), [Lmod](https://lmod.readthedocs.io) and [archspec](https://archspec.readthedocs.io). +The software layer of the EESSI project uses [EasyBuild](https://docs.easybuild.io), [Lmod](https://lmod.readthedocs.io) and [archspec](https://archspec.readthedocs.io). -See also https://eessi.github.io/docs/software_layer. +See also https://www.eessi.io/docs/software_layer . ## Pilot software stack You can set up your environment by sourcing the init script: ``` -$ source /cvmfs/pilot.eessi-hpc.org/versions/2023.06/init/bash -Found EESSI pilot repo @ /cvmfs/pilot.eessi-hpc.org/versions/2023.06! +$ source /cvmfs/software.eessi.io/versions/2023.06/init/bash +Found EESSI repo @ /cvmfs/software.eessi.io/versions/2023.06! Derived subdirectory for software layer: x86_64/intel/haswell -Using x86_64/intel/haswell subdirectory for software layer (HARDCODED) +Using x86_64/intel/haswell subdirectory for software layer Initializing Lmod... -Prepending /cvmfs/pilot.eessi-hpc.org/versions/2023.06/software/x86_64/intel/haswell/modules/all to $MODULEPATH... -Environment set up to use EESSI pilot software stack, have fun! -[EESSI pilot 2023.06] $ +Prepending /cvmfs/software.eessi.io/versions/2023.06/software/x86_64/intel/haswell/modules/all to $MODULEPATH... +Environment set up to use EESSI (2023.06), have fun! +[EESSI 2023.06] $ ``` ### Accessing EESSI via a container diff --git a/bot/build.sh b/bot/build.sh index cf9d69b65f..4af217628e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -128,12 +128,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} -# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") -echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" +export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down @@ -210,9 +210,9 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") timestamp=$(date +%s) -# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now +# to set EESSI_VERSION we need to source init/eessi_defaults now source init/eessi_defaults -export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh @@ -221,8 +221,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/bot/inspect.sh b/bot/inspect.sh index a3b88e5017..9d1fa87e1f 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -195,12 +195,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} -# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") -echo "bot/inspect.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" +export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/inspect.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down @@ -260,11 +260,11 @@ base_dir=$(dirname $(realpath $0)) # TODO better use script from tarball??? source ${base_dir}/../init/eessi_defaults -if [ -z $EESSI_PILOT_VERSION ]; then - echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 +if [ -z $EESSI_VERSION ]; then + echo "ERROR: \$EESSI_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" # NOTE The below requires access to the CVMFS repository. We could make a first # test run with a container. For now we skip the test. @@ -295,8 +295,8 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then echo "export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}" >> ${RESUME_SCRIPT} fi -if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then - echo "export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT} +if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then + echo "export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT} fi if [ ! -z ${http_proxy} ]; then echo "export http_proxy=${http_proxy}" >> ${RESUME_SCRIPT} @@ -428,7 +428,7 @@ echo "Executing command to start interactive session to inspect build job:" # TODO possibly add information on how to init session after the prefix is # entered, initialization consists of # - environment variable settings (see 'run_in_compat_layer_env.sh') -# - setup steps run in 'EESSI-pilot-install-software.sh' +# - setup steps run in 'EESSI-install-software.sh' # These initializations are combined into a single script that is executed when # the shell in startprefix is started. We set the env variable BASH_ENV here. if [[ -z ${run_in_prefix} ]]; then diff --git a/check_missing_installations.sh b/check_missing_installations.sh index e927f14701..4c04c41769 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2023.06) +# Script to check for missing installations in EESSI software stack (version 2023.06) # # author: Kenneth Hoste (@boegel) # author: Thomas Roeblitz (@trz42) diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh index 4465465082..5859dcf48a 100755 --- a/create_directory_tarballs.sh +++ b/create_directory_tarballs.sh @@ -28,9 +28,9 @@ mkdir "${tartmp}/${version}" tarname="eessi-${version}-init-$(date +%s).tar.gz" curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/init/' source "${tartmp}/${version}/init/minimal_eessi_env" -if [ "${EESSI_PILOT_VERSION}" != "${version}" ] +if [ "${EESSI_VERSION}" != "${version}" ] then - fatal_error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!" + fatal_error "Specified version ${version} does not match version ${EESSI_VERSION} in the init files!" fi tar czf "${tarname}" -C "${tartmp}" "${version}" rm -rf "${tartmp}" diff --git a/create_tarball.sh b/create_tarball.sh index 65f6efc2dc..8510caebf1 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -5,11 +5,11 @@ set -e base_dir=$(dirname $(realpath $0)) if [ $# -ne 4 ]; then - echo "ERROR: Usage: $0 " >&2 + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 -pilot_version=$2 +eessi_version=$2 cpu_arch_subdir=$3 target_tgz=$4 @@ -20,7 +20,7 @@ os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} -software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 @@ -28,7 +28,7 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 @@ -40,22 +40,22 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}.. files_list=${tmpdir}/files.list.txt module_files_list=${tmpdir}/module_files.list.txt -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' # using EasyBuild's standard module naming scheme) match the name of the @@ -64,7 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${modu # installation directories), the procedure will likely not work. for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ + ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ | grep -v '/\.wh\.' >> ${files_list} done fi diff --git a/eessi_container.sh b/eessi_container.sh index 48c4653ba9..7b0326a8b8 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -79,7 +79,7 @@ display_help() { echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script or command) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot via" + echo " repository to use [default: EESSI via" echo " default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" @@ -111,7 +111,7 @@ VERBOSE=0 STORAGE= LIST_REPOS=0 MODE="shell" -REPOSITORY="EESSI-pilot" +REPOSITORY="EESSI" RESUME= SAVE= HTTP_PROXY=${http_proxy:-} @@ -194,7 +194,7 @@ set -- "${POSITIONAL_ARGS[@]}" if [[ ${LIST_REPOS} -eq 1 ]]; then echo "Listing available repositories with format 'name [source]':" - echo " EESSI-pilot [default]" + echo " EESSI [default]" if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then cfg_load ${EESSI_REPOS_CFG_FILE} sections=$(cfg_sections) @@ -226,7 +226,7 @@ fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" fi @@ -403,7 +403,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg -if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then +if [[ "${REPOSITORY}" == "EESSI" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${TOPDIR}/init/eessi_defaults @@ -427,7 +427,7 @@ else # map { local_filepath -> container_filepath } # # repo_name_domain is the domain part of the repo_name, e.g., - # eessi-hpc.org for pilot.eessi-hpc.org + # eessi.io for software.eessi.io # # where config bundle includes the files (-> target location in container) # - default.local -> /etc/cvmfs/default.local @@ -479,7 +479,7 @@ else target=${cfg_file_map[${src}]} BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done - export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} + export EESSI_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" # need to source defaults as late as possible (after *_OVERRIDEs) source ${TOPDIR}/init/eessi_defaults diff --git a/init/Magic_Castle/bash b/init/Magic_Castle/bash index 85e4d54241..dbe1772d3a 100644 --- a/init/Magic_Castle/bash +++ b/init/Magic_Castle/bash @@ -6,7 +6,7 @@ EESSI_SILENT=1 source $(dirname "$BASH_SOURCE")/../eessi_environment_variables # Don't change the default prompt -# export PS1="[EESSI pilot $EESSI_PILOT_VERSION] $ " +# export PS1="[EESSI $EESSI_VERSION] $ " # Provide a clean MODULEPATH export MODULEPATH_ROOT=$EESSI_MODULEPATH @@ -36,4 +36,4 @@ else module reload fi -echo "Environment set up to use EESSI pilot software stack (${EESSI_PILOT_VERSION}), have fun!" +echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!" diff --git a/init/Magic_Castle/eessi_pilot_python3 b/init/Magic_Castle/eessi_python3 similarity index 73% rename from init/Magic_Castle/eessi_pilot_python3 rename to init/Magic_Castle/eessi_python3 index a762ac7b84..b2f7fd4d66 100755 --- a/init/Magic_Castle/eessi_pilot_python3 +++ b/init/Magic_Castle/eessi_python3 @@ -12,10 +12,10 @@ export EESSI_SILENT=1 # for MacOS due to the use of `readlink`) source $(dirname "$(readlink -f "$BASH_SOURCE")")/../eessi_environment_variables -eessi_pilot_python=$(ls ${EESSI_SOFTWARE_PATH}/software/Python/3*GCCcore*/bin/python | sed 1q) -if [ -f "$eessi_pilot_python" ]; then - $eessi_pilot_python "$@" +eessi_python=$(ls ${EESSI_SOFTWARE_PATH}/software/Python/3*GCCcore*/bin/python | sed 1q) +if [ -f "$eessi_python" ]; then + $eessi_python "$@" else - echo "ERROR: No EESSI pilot python 3 available." + echo "ERROR: No EESSI Python 3 available." false fi diff --git a/init/bash b/init/bash index ea605db0b5..84fe783bce 100644 --- a/init/bash +++ b/init/bash @@ -13,7 +13,7 @@ source $(dirname "$BASH_SOURCE")/eessi_environment_variables # only continue if setting EESSI environment variables worked fine if [ $? -eq 0 ]; then - export PS1="[EESSI pilot $EESSI_PILOT_VERSION] $ " + export PS1="[EESSI $EESSI_VERSION] $ " # add location of commands provided by compat layer to $PATH; # see https://github.com/EESSI/software-layer/issues/52 @@ -28,12 +28,12 @@ if [ $? -eq 0 ]; then module use $EESSI_MODULEPATH #echo >> $output - #echo "*** Known problems in the ${EESSI_PILOT_VERSION} pilot software stack ***" >> $output + #echo "*** Known problems in the ${EESSI_VERSION} software stack ***" >> $output #echo >> $output #echo "1) ..." >> $output #echo >> $output #echo >> $output - echo "Environment set up to use EESSI pilot software stack, have fun!" >> $output + echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!" fi diff --git a/init/eessi_defaults b/init/eessi_defaults index 0143dc38ab..50533ac932 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -8,5 +8,5 @@ # license: GPLv2 # -export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}" +export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}" +export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 10ac1926f4..18d8e70906 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -15,11 +15,11 @@ function error() { false } -# set up minimal environment: $EESSI_PREFIX, $EESSI_PILOT_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX +# set up minimal environment: $EESSI_PREFIX, $EESSI_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX source $EESSI_INIT_DIR_PATH/minimal_eessi_env if [ -d $EESSI_PREFIX ]; then - echo "Found EESSI pilot repo @ $EESSI_PREFIX!" >> $output + echo "Found EESSI repo @ $EESSI_PREFIX!" >> $output export EESSI_EPREFIX=$EPREFIX if [ -d $EESSI_EPREFIX ]; then @@ -82,5 +82,5 @@ if [ -d $EESSI_PREFIX ]; then error "Compatibility layer directory $EESSI_EPREFIX not found!" fi else - error "EESSI pilot repository at $EESSI_PREFIX not found!" + error "EESSI repository at $EESSI_PREFIX not found!" fi diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index b7cb7c5e9e..5b3562068d 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -4,11 +4,11 @@ # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) -# set up defaults: EESSI_CVMFS_REPO, EESSI_PILOT_VERSION +# set up defaults: EESSI_CVMFS_REPO, EESSI_VERSION # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_PILOT_VERSION +export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' diff --git a/install_software_layer.sh b/install_software_layer.sh index bf3006a4a0..82ca70b73f 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,4 +1,4 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -./run_in_compat_layer_env.sh ./EESSI-pilot-install-software.sh "$@" +./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 12688e2aed..f57c4d0749 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -3,11 +3,11 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -if [ -z $EESSI_PILOT_VERSION ]; then - echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 +if [ -z $EESSI_VERSION ]; then + echo "ERROR: \$EESSI_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 exit 1 @@ -23,8 +23,8 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" fi -if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then - INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}" fi if [ ! -z ${http_proxy} ]; then INPUT="export http_proxy=${http_proxy}; ${INPUT}" @@ -33,5 +33,5 @@ if [ ! -z ${https_proxy} ]; then INPUT="export https_proxy=${https_proxy}; ${INPUT}" fi -echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From a72b0778286a43cf928acf45d311e37ed7265b16 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Nov 2023 22:16:33 +0100 Subject: [PATCH 2/6] add initial easystack file for software.eessi.io + move 'eb --show-config' command to where eb command is available --- EESSI-install-software.sh | 6 +++--- .../2023.06/eessi-2023.06-eb-4.8.2-system.yml | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 234f56d518..f6087b3cfe 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -172,8 +172,6 @@ if [ ! -z "${shared_fs_path}" ]; then export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} fi -${EB} --show-config - echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge @@ -190,7 +188,7 @@ fi pr_diff=$(ls [0-9]*.diff | head -1) # use PR patch file to determine in which easystack files stuff was added -for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$' | egrep -v 'known-issues|missing'); do +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do echo -e "Processing easystack file ${easystack_file}...\n\n" @@ -200,6 +198,8 @@ for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[ # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + ${EB} --show-config + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." if [ -f ${easystack_file} ]; then diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml new file mode 100644 index 0000000000..5ce6a65913 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.8.2.eb: + options: + from-pr: 19105 From 4f3691be0f85f7b313c887c3b770bc837fd11518 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Nov 2023 22:30:16 +0100 Subject: [PATCH 3/6] add CI workflow for software.eessi.io + use software.eessi.io in tests for eessi_container.sh script --- ...{test_eessi.yml => test_eessi-hpc.org.yml} | 2 +- .github/workflows/test_eessi.io.yml | 76 +++++++++++++++++++ .../workflows/test_eessi_container_script.yml | 12 +-- 3 files changed, 83 insertions(+), 7 deletions(-) rename .github/workflows/{test_eessi.yml => test_eessi-hpc.org.yml} (98%) create mode 100644 .github/workflows/test_eessi.io.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi-hpc.org.yml similarity index 98% rename from .github/workflows/test_eessi.yml rename to .github/workflows/test_eessi-hpc.org.yml index 6dfd78e428..3e3d7cca7b 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi-hpc.org.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests relying on having EESSI pilot repo mounted +name: Tests relying on having pilot.eessi-hpc.org mounted on: [push, pull_request, workflow_dispatch] permissions: contents: read # to fetch code (actions/checkout) diff --git a/.github/workflows/test_eessi.io.yml b/.github/workflows/test_eessi.io.yml new file mode 100644 index 0000000000..e2ee713765 --- /dev/null +++ b/.github/workflows/test_eessi.io.yml @@ -0,0 +1,76 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests relying on having software.eessi.io mounted +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + pilot: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR: + - aarch64/generic + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/generic + steps: + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + with: + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: Test check_missing_installations.sh script + run: | + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}*.yml)" + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done + + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) + run: | + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + # create dummy easystack file with a single entry (something that is not installed in EESSI) + easystack_file="test.yml" + echo "easyconfigs:" > ${easystack_file} + echo " - GCC-8.3.0:" >> ${easystack_file} + echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):" + cat ${easystack_file} + # note, check_missing_installations.sh exits 1 if a package was + # missing, which is intepreted as false (exit code based, not + # boolean logic), hence when the script exits 0 if no package was + # missing it is interpreted as true, thus the test did not capture + # the missing package + if ./check_missing_installations.sh ${easystack_file}; then + echo "did NOT capture missing package; test FAILED" + exit 1 + else + echo "captured missing package; test PASSED" + exit 0 + fi diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 929fb22cec..590aa4225b 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -90,15 +90,15 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then outfile=out_readwrite.txt fn="test_${RANDOM}.txt" - echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh chmod u+x test_script.sh export SINGULARITY_BIND="$PWD:/test" ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") # note: must use '--access rw' again here, since touched file is in overlay upper dir - ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then @@ -120,12 +120,12 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then outfile=out_save.txt fn="test_${RANDOM}.txt" - test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + test_cmd="touch /cvmfs/software.eessi.io/${fn}" ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} rm -f ${outfile} - ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile tar tfv test-save.tar | grep "overlay-upper/${fn}" From 3a9f86d338abf396f2413508e9ed6510bf9fd106 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Nov 2023 17:14:08 +0100 Subject: [PATCH 4/6] use archdetect by default for software.eessi.io version 2023.06 --- init/eessi_defaults | 3 +++ init/eessi_environment_variables | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/init/eessi_defaults b/init/eessi_defaults index 50533ac932..2424a3ce97 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -10,3 +10,6 @@ export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}" export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" +# use archdetect by default, unless otherwise specified +export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=0}" +export EESSI_USE_ARCHSPEC="${EESSI_USE_ARCHSPEC:=1}" diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 18d8e70906..42f4b6b76a 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -29,11 +29,13 @@ if [ -d $EESSI_PREFIX ]; then # if archdetect is enabled, use internal code export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath) echo "archdetect says ${EESSI_SOFTWARE_SUBDIR}" >> $output - else + elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined! export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3 export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX) echo "archspec says ${EESSI_SOFTWARE_SUBDIR}" >> $output + else + error "Don't know how to detect host CPU, giving up!" fi if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then From fe65d9ec6a729e3f940c14363e66abaa25aedbe6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Nov 2023 17:18:31 +0100 Subject: [PATCH 5/6] add debugging to CI workflow to test scripts --- .github/workflows/tests_scripts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 74e2ebcffe..a8bbf9d762 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -54,6 +54,8 @@ jobs: # both $EB and $TMPDIR environment must be set, required by load_easybuild_module.sh script echo 'export EB="eb"' >> ${test_script} echo 'export TMPDIR=$(mktemp -d)' >> ${test_script} + # debugging: check contents of CernVM-FS config repo for traces of EESSI (for software.eessi.io) + echo 'find /cvmfs/cvmfs-config.cern.ch -type f | grep eessi | xargs sha256sum' >> ${test_script} # set up environment to have utility functions in place that load_easybuild_module.sh script relies on, # along with $EESSI_* environment variables, and Lmod echo 'ls -l /software-layer/' >> ${test_script} From 2e4a43b8b7f657fa8f37094dcea057908f454e92 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Nov 2023 19:01:46 +0100 Subject: [PATCH 6/6] always let eessi_container.sh mount cvmfs-config repo to get access to software.eessi.io --- eessi_container.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 7b0326a8b8..838e05923c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -513,6 +513,10 @@ fi # 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() + +# always mount cvmfs-config repo (to get access to software.eessi.io) +EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") + if [[ "${ACCESS}" == "ro" ]]; then export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}"