diff --git a/configs/local.yaml b/configs/local.yaml index f42b0fe..f872f05 100644 --- a/configs/local.yaml +++ b/configs/local.yaml @@ -33,9 +33,18 @@ force: conda_arch: 'Linux-x86_64' max_num_cpus: 4 # No logic currently to refer to other variables here - # So either implement that, throw them to $TMPDIR - # + # So either implement that, throw them to $TMPDIR workdir: $PWD + # What executable name or path to use for singularity + # For fully dynamic behavior + # You can just set singularity or apptainer here. + # Keep in mind that this could break installations + # if e.g the user installs singularity via conda + # when user namespaces are not enabled + # Environment variables can be used here + # but they will be expanded during build, + # not during runtime + singularity_executable_path: "singularity" # Value will override any user set value diff --git a/configs/lumi.yaml b/configs/lumi.yaml index 0e57bdb..5962010 100644 --- a/configs/lumi.yaml +++ b/configs/lumi.yaml @@ -41,6 +41,16 @@ force: # So either implement that, throw them to $TMPDIR # workdir: $PWD + # What executable name or path to use for singularity + # For fully dynamic behavior + # You can just set singularity or apptainer here. + # Keep in mind that this could break installations + # if e.g the user installs singularity via conda + # when user namespaces are not enabled + # Environment variables can be used here + # but they will be expanded during build, + # not during runtime + singularity_executable_path: "/usr/bin/singularity" # Value will override any user set value diff --git a/configs/mahti.yaml b/configs/mahti.yaml index 5a05098..d1cfc7a 100644 --- a/configs/mahti.yaml +++ b/configs/mahti.yaml @@ -39,6 +39,16 @@ force: # So either implement that, throw them to $TMPDIR # workdir: $PWD + # What executable name or path to use for singularity + # For fully dynamic behavior + # You can just set singularity or apptainer here. + # Keep in mind that this could break installations + # if e.g the user installs singularity via conda + # when user namespaces are not enabled + # Environment variables can be used here + # but they will be expanded during build, + # not during runtime + singularity_executable_path: "/usr/bin/singularity" # Value will override any user set value diff --git a/configs/puhti.yaml b/configs/puhti.yaml index a1f7729..58b5a0a 100644 --- a/configs/puhti.yaml +++ b/configs/puhti.yaml @@ -40,6 +40,16 @@ force: # So either implement that, throw them to $TMPDIR # workdir: $PWD + # What executable name or path to use for singularity + # For fully dynamic behavior + # You can just set singularity or apptainer here. + # Keep in mind that this could break installations + # if e.g the user installs singularity via conda + # when user namespaces are not enabled + # Environment variables can be used here + # but they will be expanded during build, + # not during runtime + singularity_executable_path: "/usr/bin/singularity" # Value will override any user set value diff --git a/create_inst.sh b/create_inst.sh index d62beed..bb55247 100755 --- a/create_inst.sh +++ b/create_inst.sh @@ -54,18 +54,18 @@ export install_root=$CW_INSTALLATION_PATH if [[ "$CW_UPDATE_INSTALLATION" == "yes" ]];then - _CONTAINER_EXEC="singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE" + _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_SOURCE_MOUNT_POINT:image-src=/ _deploy/$CW_CONTAINER_IMAGE" export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$_inst_path/_bin:$_inst_path/bin" print_info "Copying installation to writable area, might take a while" 1 print_info "$(readlink -f $CW_INSTALLATION_PREFIX)" 1 $_CONTAINER_EXEC cp -a $CW_SOURCE_MOUNT_POINT/. $CW_INSTALLATION_PATH || { print_err "Failed to copy some files, most likely incorrect file permissions inside the squashfs image" && false ; } elif [[ "$CW_MODE" == "wrapdisk" ]];then export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH,$CW_WRAP_SRC:$CW_SOURCE_MOUNT_POINT" - _CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE" + _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE" else export SINGULARITY_BIND="$SINGULARITY_BIND,$PWD/_inst_dir:$CW_INSTALLATION_PATH" - _CONTAINER_EXEC="singularity --silent exec _deploy/$CW_CONTAINER_IMAGE" + _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE" fi cp ./_sing_inst_script.sh _pre_install.sh _post_install.sh _inst_dir print_info "Running installation script" 1 diff --git a/frontends/containerize b/frontends/containerize index a0fe439..871ce37 100755 --- a/frontends/containerize +++ b/frontends/containerize @@ -98,6 +98,19 @@ else export _inst_path=$(realpath -s $PWD/$CW_INSTALLATION_PREFIX ) fi +if ! command -v $CW_SINGULARITY_EXECUTABLE_PATH &>/dev/null; then + { print_err "The command $CW_SINGULARITY_EXECUTABLE_PATH does not exists + Verify that the value for 'singularity_executable_path' in $CW_GLOBAL_YAML is correct"; false ;} +fi + +print_info "Validating the set singularity command $CW_SINGULARITY_EXECUTABLE_PATH" 2 +num_res=$($CW_SINGULARITY_EXECUTABLE_PATH --help | grep "build\|exec\|shell" 2>/dev/null | wc -l) +if [[ "$num_res" -lt 3 ]];then + { print_err "$CW_SINGULARITY_EXECUTABLE_PATH does not seem to be a valid apptainer/singularity executable"; false ;} +fi + + + $M_SCRIPT_DIR/../pre.sh || { print_err "Failed getting container image"; false ;} # Nothing to do if we are just wrapping a container if [[ ! "$CW_MODE" == "wrapcont" ]];then diff --git a/generate_wrappers.sh b/generate_wrappers.sh index 467edd9..78134dd 100755 --- a/generate_wrappers.sh +++ b/generate_wrappers.sh @@ -15,14 +15,14 @@ echo "#!/bin/bash" > _deploy/common.sh if [[ "$CW_MODE" == "wrapcont" ]];then - _CONTAINER_EXEC="/usr/bin/singularity --silent exec _deploy/$CW_CONTAINER_IMAGE" - _RUN_CMD="/usr/bin/singularity --silent exec \$DIR/../\$CONTAINER_IMAGE" - _SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE" + _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec _deploy/$CW_CONTAINER_IMAGE" + _RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec \$DIR/../\$CONTAINER_IMAGE" + _SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE" else - _CONTAINER_EXEC="/usr/bin/singularity --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE" + _CONTAINER_EXEC="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec -B _deploy/$CW_SQFS_IMAGE:$CW_INSTALLATION_PATH:image-src=/ _deploy/$CW_CONTAINER_IMAGE" echo "SQFS_IMAGE=$CW_SQFS_IMAGE" >> _deploy/common.sh - _RUN_CMD="/usr/bin/singularity --silent exec \$DIR/../\$CONTAINER_IMAGE" - _SHELL_CMD="/usr/bin/singularity --silent shell \$DIR/../\$CONTAINER_IMAGE" + _RUN_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent exec \$DIR/../\$CONTAINER_IMAGE" + _SHELL_CMD="$CW_SINGULARITY_EXECUTABLE_PATH --silent shell \$DIR/../\$CONTAINER_IMAGE" fi # Need to unset the path, otherwise we might be stuck in a nasty loop diff --git a/pre.sh b/pre.sh index b2ecd2f..e47edd1 100755 --- a/pre.sh +++ b/pre.sh @@ -14,8 +14,11 @@ mkdir -p $CW_BUILD_TMPDIR/_deploy/ if [[ ! -e $CW_CONTAINER_SRC ]]; then print_info "Fetching container $CW_CONTAINER_SRC" 1 - singularity --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \ + $CW_SINGULARITY_EXECUTABLE_PATH --silent pull $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE $CW_CONTAINER_SRC || \ { print_err "Failed fetching container"; exit 1 ;} + if [[ ! -e $CW_BUILD_TMPDIR/_deploy/$CW_CONTAINER_IMAGE ]];then + { print_err "$CW_SINGULARITY_EXECUTABLE_PATH pull returned success, but no container was fetched"; exit 1 ;} + fi else if [[ ${CW_SHARE_CONTAINER+defined} && ${CW_SHARE_CONTAINER} == "yes" ]];then print_info "Using container $CW_CONTAINER_SRC" 1 diff --git a/tests/test-03.sh b/tests/test-03.sh index e57c163..294c82b 100644 --- a/tests/test-03.sh +++ b/tests/test-03.sh @@ -32,6 +32,19 @@ rule ghelp: """ ' > Snakefile +cat ../../default_config/config.yaml | sed "s@singularity_executable_path.*@singularity_executable_path: 'ThisIsNotACommand'@g" > my_config.yaml +export CW_GLOBAL_YAML=$( readlink -f my_config.yaml) +t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'ThisIsNotACommand does not exists'" "Exit if configured singularity command does not exist" +cat ../../default_config/config.yaml | sed "s@singularity_executable_path.*@singularity_executable_path: 'my_sing_command'@g" > my_config.yaml +echo "#!/bin/bash" > my_sing_command +echo "exit 0" >> my_sing_command +chmod +x my_sing_command +export PATH=$PATH:$PWD +t_run "conda-containerize new --mamba env2.yml --prefix S | grep 'does not seem to be a valid apptainer/singularity executable'" "Exit if configured singularity command seems broken" +# Run the rest of the test with singularity found from path. +cat ../../default_config/config.yaml | sed "s@singularity_executable_path.*@singularity_executable_path: 'singularity'@g" > my_config.yaml + + t_run "conda-containerize new --mamba env2.yml --prefix S" "mamba works" t_run "conda-containerize new --mamba env.yml --prefix Gdal" "gdal installed" export OPATH=$PATH