diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index a2fb0412d8..4693e147ab 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -44,7 +44,7 @@ jobs: export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/repos ./eessi_container.sh --list-repos - - name: test load_easybuild_module.sh script + - name: test load_easybuild_module.sh script with software.eessi.io run: | # bind current directory into container as /software-layer export SINGULARITY_BIND="${PWD}:/software-layer" @@ -79,6 +79,50 @@ jobs: chmod u+x ${test_script} + # run wrapper script + capture & check output + out="${PWD}/eb-${EB_VERSION}.out" + export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/repos + export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/software.eessi.io" + ./eessi_container.sh --access rw --mode run --verbose --repository EESSI /software-layer/run_in_compat_layer_env.sh /software-layer/eb-${EB_VERSION}.sh 2>&1 | tee ${out} + pattern="^This is EasyBuild ${EB_VERSION} " + grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) + done + + - name: test load_easybuild_module.sh script + run: | + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" + + # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06, + # since Python in compat layer is Python 3.11.x; + # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version + for EB_VERSION in '4.8.0'; do + # Create script that uses load_easybuild_module.sh which we can run in compat layer environment + # note: Be careful with single vs double quotes below! + # ${EB_VERSION} should be expanded, so use double quotes; + # For statements using variables that are only defined in the script, like ${EASYBUILD_INSTALLPATH}, + # use single quotes to avoid expansion while creating the script. + test_script="${PWD}/eb-${EB_VERSION}.sh" + echo '#!/bin/bash' > ${test_script} + # both $EB and $TMPDIR environment must be set, required by load_easybuild_module.sh script + echo 'export EB="eb"' >> ${test_script} + echo 'export TMPDIR=$(mktemp -d)' >> ${test_script} + # set up environment to have utility functions in place that load_easybuild_module.sh script relies on, + # along with $EESSI_* environment variables, and Lmod + echo 'ls -l /software-layer/' >> ${test_script} + echo 'source /software-layer/scripts/utils.sh' >> ${test_script} + echo 'source /software-layer/init/eessi_environment_variables' >> ${test_script} + echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script} + # minimal configuration for EasyBuild so we can test installation aspect of load_easybuild_module.sh script + echo "export EASYBUILD_INSTALLPATH=/tmp/eb-${EB_VERSION}" >> ${test_script} + echo 'module use ${EASYBUILD_INSTALLPATH}/modules/all' >> ${test_script} + echo '' >> ${test_script} + echo "source /software-layer/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script} + echo 'module list' >> ${test_script} + echo 'eb --version' >> ${test_script} + + chmod u+x ${test_script} + # run wrapper script + capture & check output out="${PWD}/eb-${EB_VERSION}.out" export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/repos diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index eb616b1311..df47df4e9c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -143,7 +143,8 @@ fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables +# EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables +EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then fatal_error "Failed to determine software subdirectory?!" diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml new file mode 100644 index 0000000000..a7bce002c7 --- /dev/null +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.9.0.eb: + options: + from-pr: 19464 diff --git a/eessi_container.sh b/eessi_container.sh index 6e68524edb..149387f4ad 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -576,6 +576,7 @@ declare -a EESSI_FUSE_MOUNTS=() # always mount cvmfs-config repo (to get access to software.eessi.io) # Commented out intentionally EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") +EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") if [[ "${ACCESS}" == "ro" ]]; then export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh index d1bfd18bb5..e8fd762c13 100755 --- a/load_easybuild_module.sh +++ b/load_easybuild_module.sh @@ -55,7 +55,15 @@ else EB_TMPDIR=${TMPDIR}/ebtmp echo ">> Temporary installation (in ${EB_TMPDIR})..." pip_install_out=${TMPDIR}/pip_install.out + which pip3 pip3 install --prefix ${EB_TMPDIR} easybuild &> ${pip_install_out} + if [[ $? -ne 0 ]]; then + echo "pip3 installing easybuild failed; see command output below" + cat ${pip_install_out} + else + echo "pip3 installing easybuild succeeded; see command output below" + cat ${pip_install_out} + fi # keep track of original $PATH and $PYTHONPATH values, so we can restore them ORIG_PATH=${PATH} @@ -67,8 +75,23 @@ else eb_install_out=${TMPDIR}/eb_install.out ok_msg="Latest EasyBuild release installed, let's go!" fail_msg="Installing latest EasyBuild release failed, that's not good... (output: ${eb_install_out})" - ${EB} --install-latest-eb-release 2>&1 | tee ${eb_install_out} - check_exit_code $? "${ok_msg}" "${fail_msg}" + which ${EB} + ${EB} --debug --install-latest-eb-release 2>&1 | tee ${eb_install_out} + exit_code=$? + if [[ $exit_code -ne 0 ]]; then + # show last log + echo "show last log: cat $(${EB} --last-log)" + cat $(${EB} --last-log) + + # retry: wait a minute, then rerun command + date + SLEEP=60 + echo "first attempt failed, retrying it after waiting $SLEEP seconds" + sleep $SLEEP + date + ${EB} --debug --install-latest-eb-release 2>&1 | tee -a ${eb_install_out} + fi + check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" # maybe the module obtained with --install-latest-eb-release is exactly the EasyBuild version we wanted? IGNORE_CACHE=''