From 66eff18ee5f8be2e1e7ce3fe07bd8f77bb65c623 Mon Sep 17 00:00:00 2001 From: Jess <20195932+wrongkindofdoctor@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:44:36 -0500 Subject: [PATCH] Fix install scripts (#493) * Create PULL_REQUEST_TEMPLATE.md * Delete PULL_REQUEST_TEMPLATE.md * add debugging lines to conda_env_setup fix conda setup block in conda_init.sh * remove space in --prefix call in conda_env_setup.sh * refine conda initialization script * remove some echo statements * change conda_check procedure in conda_init.sh * add conda binary check back to init script * fix issues with the enviroment scripts add option to specify the micromamba executable path to cli config files, env build scripts, and docs add help option to conda_env_setup.sh simplify conda env command in environment_manager module add arg to output_manager to try and fix issues with ghostscript converting files on personal mac update docs with description on micromamba_exe option * add micromamba_exe parm to conda_init.sh update workflow yml with new micromamba args add micromamba_exe to test workflow config files and update conda_root to reflect changes * remove PathAction spec from micromamba_exe in cli_subcommands template since it is an optional argument and does not require initialization by default * remove extraneous calls from conda_init.sh * remove more debugging calls from conda_init.sh * add logic to handle the presence of optional micromamba_exe path to core path initialization * fix mdtf generator for micromamba case in conda_env_setup * try adding source conda_init.sh and activate _MDTF_base in output_managerto resolve ghostscript issues * add ghostscript to env_base.yml * move micromamba environment setup scripts to separate files and reinstate previous code in conda env setup and init files update code to reflect changes in conda/micromamba scripts update mdtf_framework.py to allow python 3.10 or later in case users need python 3.10 environments * update docs and README conda installation instructions and options for micromamba uncomment cat output file in mdtf_tests for debugging * fix actions file * replace calls to mdtf_framework.py with mdtf wrapper in mdtf_tests.yml * add micromamba_exe arg to NOAA_GFDL runtime config templates remove env_mgr parm from make_output remove conda script calls from output manager * update conda/micromamba instructions in start_install docs --------- Co-authored-by: Jessica.Liptak --- .github/workflows/mdtf_tests.yml | 16 +- README.md | 14 +- .../multirun_config_template.jsonc | 2 + doc/sphinx/dev_start.rst | 13 +- doc/sphinx/ref_cli.rst | 9 +- doc/sphinx/start_install.rst | 30 ++- mdtf_framework.py | 4 +- sites/NOAA_GFDL/default_gfdl.jsonc | 10 +- .../gfdl_multirun_config_template.jsonc | 3 + src/cli_plugins.jsonc | 5 + src/conda/conda_env_setup.sh | 124 ++++------- src/conda/conda_init.sh | 25 +-- src/conda/env_base.yml | 1 + src/conda/micromamba_env_setup.sh | 207 ++++++++++++++++++ src/conda/micromamba_init.sh | 165 ++++++++++++++ src/core.py | 4 + src/default_tests.jsonc | 5 +- src/environment_manager.py | 25 +-- src/output_manager.py | 9 +- tests/github_actions_test_macos_set1.jsonc | 5 +- tests/github_actions_test_macos_set2.jsonc | 4 +- tests/github_actions_test_macos_set3.jsonc | 4 +- tests/github_actions_test_ubuntu_set1.jsonc | 3 +- tests/github_actions_test_ubuntu_set2.jsonc | 4 +- tests/github_actions_test_ubuntu_set3.jsonc | 3 +- 25 files changed, 529 insertions(+), 165 deletions(-) create mode 100755 src/conda/micromamba_env_setup.sh create mode 100755 src/conda/micromamba_init.sh diff --git a/.github/workflows/mdtf_tests.yml b/.github/workflows/mdtf_tests.yml index 85367909e..b93ac93bb 100644 --- a/.github/workflows/mdtf_tests.yml +++ b/.github/workflows/mdtf_tests.yml @@ -68,7 +68,8 @@ jobs: run: | echo "Installing XQuartz" brew install --cask xquartz - echo "CONDA_ROOT=$(echo /Users/runner/micromamba-bin)" >> $GITHUB_ENV + echo "CONDA_ROOT=$(echo /Users/runner/micromamba)" >> $GITHUB_ENV + echo "MICROMAMBA_EXE=$(echo /Users/runner/micromamba-bin/micromamba)" >> $GITHUB_ENV echo "CONDA_ENV_DIR=$(echo /Users/runner/micromamba/envs)" >> $GITHUB_ENV - name: Set environment variables run: | @@ -76,7 +77,8 @@ jobs: - name: Set conda vars if: ${{ matrix.os == 'ubuntu-latest' }} run: | - echo "CONDA_ROOT=$(echo /home/runner/micromamba-bin)" >> $GITHUB_ENV + echo "MICROMAMBA_EXE=$(echo /home/runner/micromamba-bin/micromamba)" >> $GITHUB_ENV + echo "CONDA_ROOT=$(echo /home/runner/micromamba)" >> $GITHUB_ENV echo "CONDA_ENV_DIR=$(echo /home/runner/micromamba/envs)" >> $GITHUB_ENV - name: Install Conda Environments run: | @@ -84,7 +86,7 @@ jobs: echo "conda root ${CONDA_ROOT}" echo "env dir ${CONDA_ENV_DIR}" # MDTF-specific setup: install all conda envs - ./src/conda/conda_env_setup.sh --all --micromamba_root ${CONDA_ROOT} --env_dir ${CONDA_ENV_DIR} + ./src/conda/micromamba_env_setup.sh --all --micromamba_root ${CONDA_ROOT} --micromamba_exe ${MICROMAMBA_EXE} --env_dir ${CONDA_ENV_DIR} echo "Creating the _MDTF_synthetic_data environment" micromamba create -y -f ./src/conda/_env_synthetic_data.yml - name: Generate Model Data @@ -159,9 +161,9 @@ jobs: run: | micromamba activate _MDTF_base # run the test PODs - ./mdtf_framework.py -v -f ${{matrix.json-file-set2}} - # Uncomment the following line for debugging - # cat ../wkdir/MDTF_GFDL.Synthetic_1_10/MJO_prop_amp/MJO_prop_amp.log + ./mdtf -v -f ${{matrix.json-file-set2}} + # Uncomment the following line for debugging + #cat ../wkdir/MDTF_GFDL.Synthetic_1_10/MJO_prop_amp/MJO_prop_amp.log - name: Get observational data for set 3 run: | echo "${PWD}" @@ -198,7 +200,7 @@ jobs: run: | micromamba activate _MDTF_base # run the test PODs - ./mdtf_framework.py -v -f ${{matrix.json-file-set3}} + ./mdtf -v -f ${{matrix.json-file-set3}} - name: Run unit tests run: | micromamba activate _MDTF_base diff --git a/README.md b/README.md index ac0a9a5d7..0af8b335d 100644 --- a/README.md +++ b/README.md @@ -87,14 +87,18 @@ packages do not support them at this time. ### ANACONADA/MINICONDA `% ./src/conda/conda_env_setup.sh --all --conda_root $CONDA_ROOT --env_dir $CONDA_ENV_DIR` ### MICROMAMBA on machines that do NOT have Apple M-series chips -`% ./src/conda/conda_env_setup.sh --all --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR` +`% ./src/conda/micromamba_env_setup.sh --all --micromamba_root $MICROMAMBA_ROOT --micromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR` ### MICROMAMBA on machines with Apple M-series chips -`% ./src/conda/conda_env_setup.sh --e base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR` +`% ./src/conda/micromamba_env_setup.sh --e base --micromamba_root $MICROMAMBA_ROOT --micromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR` -`% ./src/conda/conda_env_setup.sh --e python3_base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR` +`% ./src/conda/micromamba_env_setup.sh --e python3_base --micromamba_root $MICROMAMBA_ROOT ---micromamba_exe $MICROMAMBA_EXE -env_dir $CONDA_ENV_DIR` - - Substitute the actual paths for `$CODE_ROOT`, `$CONDA_ROOT`, `$MICROMAMBA_ROOT`, and `$CONDA_ENV_DIR`. - -`$MICROMAMBA_ROOT` is location of the micromamba executable on your machine (e.g., /home/${USER}/.local/bin) + - Substitute the actual paths for `$CODE_ROOT`, `$CONDA_ROOT`, `$MICROMAMBA_ROOT`, `MICROMAMBA_EXE`, and `$CONDA_ENV_DIR`. + - `$MICROMAMBA_ROOT` is the path to micromamba installation on your system + (e.g., /home/${USER}/micromamba). This is defined by the `$MAMBA_ROOT_PREFIX` environment variable on your system + when micromamba is installed + - `$MICROMAMBA_EXE` is full path to the micromamba executable on your system + (e.g., /home/${USER}/.local/bin/micromamba). This is defined by the `MAMBA_EXE` environment variable on your system - The `--env_dir` flag allows you to put the program files in a designated location `$CONDA_ENV_DIR` (for space reasons, or if you don’t have write access). You can omit this flag, and the environments will be installed within `$CONDA_ROOT/envs/` by default. diff --git a/diagnostics/example_multicase/multirun_config_template.jsonc b/diagnostics/example_multicase/multirun_config_template.jsonc index a5e7fb6bf..ad5a33d94 100644 --- a/diagnostics/example_multicase/multirun_config_template.jsonc +++ b/diagnostics/example_multicase/multirun_config_template.jsonc @@ -73,6 +73,8 @@ // location. "conda_env_root": "", + // Path to micromamba executable if using micromamba + "micromamba_exe": "", // SETTINGS ------------------------------------------------------------------ // Any command-line option recognized by the mdtf script (type `mdtf --help`) // can be set here, in the form "flag name": "desired setting". diff --git a/doc/sphinx/dev_start.rst b/doc/sphinx/dev_start.rst index 834ed556b..bda722b37 100644 --- a/doc/sphinx/dev_start.rst +++ b/doc/sphinx/dev_start.rst @@ -34,6 +34,7 @@ Regardless of development language, we strongly recommend that developers use co Python-based PODs should be written in Python 3.11 or newer. We provide a developer version of the python3_base environment (described below) that includes Jupyter and other developer-specific tools. This is not installed by default, and must be requested by passing the ``--all`` flag to the conda setup script: +If you are using Anaconda or miniconda to manage the conda environments, run: .. code-block:: console % cd $CODE_ROOT @@ -45,17 +46,19 @@ Installing dependencies via Micromamba Micromamba is a lightweight version of Anaconda. It is required to install the base and python3_base conda enviroments on macOS machines with Apple M-series chips. Installation instructions are available in the `Micromamba Documentation `__, Once Micromamba is installed on your system, run the following to install all conda environments if you are NOT using an -Apple M-series machine, where `$MICROMAMBA_ROOT` is the location of the micromamba executable on your system: +Apple M-series machine, where `$MICROMAMBA_ROOT` is the location of the micromamba installation, and +`MICROMAMBA_EXE` is the path to the micromamba executable on your system: .. code-block:: console % cd $CODE_ROOT - % ./src/conda/conda_env_setup.sh --all --conda_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR + % ./src/conda/micromamba_env_setup.sh --all --conda_root $MICROMAMBA_ROOT --micromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR + +If you are using an Apple M-series machine, you can install just the base and python3_base environments: -If you are using an Apple M-series machine, you can install just the base and python3_base environnents: .. code-block:: console - % ./src/conda/conda_env_setup.sh -e base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR - % ./src/conda/conda_env_setup.sh -e python3_base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR + % ./src/conda/micromamba_env_setup.sh -e base --micromamba_root $MICROMAMBA_ROOT --mircromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR + % ./src/conda/micromamba_env_setup.sh -e python3_base --micromamba_root $MICROMAMBA_ROOT --mircromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR POD development using existing Conda environments ------------------------------------------------- diff --git a/doc/sphinx/ref_cli.rst b/doc/sphinx/ref_cli.rst index 7277f1e63..2e9ff631e 100644 --- a/doc/sphinx/ref_cli.rst +++ b/doc/sphinx/ref_cli.rst @@ -61,6 +61,7 @@ General options | | The default value for this setting is ``local``. The sites/local/ directory is left empty in order to enable any installation to be customized (e.g. settings the paths to where supporting data was installed) without needing to alter the framework code. For more information on how to do this, see the documentation for the `'local' site <../sphinx_sites/local.html>`__. + -f, --input-file Path to a user configuration file that sets options listed here. This can be a JSON file of the form given in `src/default_tests.jsonc `__ (which is intended to be copied and used as a template), or a text file containing flags and command-line arguments as they would be entered in the shell. Additional options set explicitly on the command line will still override settings in this file. Path settings @@ -99,6 +100,13 @@ Options that describe the input model data and how it should be obtained. | that analyze output from a single model simulation and an (optional) observational dataset | Use ``"multi_run"`` for PODs that analyze output from 2 or more model simulations and/or observational datasets (cases). | See the example_multicase POD and config files for an example of a ``multi_run`` type POD. + +Conda/micromamba settings ++++++++++++++++++++++++++ +--conda_root path to anaconda, miniconda, or micromamba installation +--conda_env_root path to directory with conda enviroments +--micromamba_exe path to the micromamba executable. REQUIRED if using micromamba + Analysis settings +++++++++++++++++ @@ -130,7 +138,6 @@ Options that control how the package is deployed (how code dependencies are mana .. note:: The values used for this option and its settings must be compatible with how the package was set up during :doc:`installation`. Missing code dependencies are not installed at runtime; instead any POD with missing dependencies raises an error and is not run. - Output options ++++++++++++++ diff --git a/doc/sphinx/start_install.rst b/doc/sphinx/start_install.rst index 6bff3819e..744aa15cb 100644 --- a/doc/sphinx/start_install.rst +++ b/doc/sphinx/start_install.rst @@ -142,7 +142,7 @@ this installation needs to be done as a manual step. Managing dependencies with the conda package manager ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The MDTF framework code is written in Python 3.10, +The MDTF framework code is written in Python 3.11, but supports running PODs written in a variety of scripting languages and combinations of libraries. To ensure that the correct versions of these dependencies are installed and available, we use `conda `__, a free, open-source package manager. @@ -157,8 +157,8 @@ flags described below. Users may install their own copies of Anaconda/Miniconda on their machine, or use a centrally-installed version managed by their institution. Note that installing your own copy of Anaconda/Miniconda -will re-define the default locations of the conda executable and environment directory defined in your `.bashrc` or -`.cshrc` file if you have previously used a version of conda managed by your institution, +will re-define the default locations of the conda executable and environment directory defined in your `.bash_profile`, +.bashrc`, or `.cshrc` file if you have previously used a version of conda managed by your institution, so you will have to re-create any environments made using central conda installations. If these space requirements are prohibitive, we provide an alternate method of installation which makes @@ -205,15 +205,24 @@ Installing the package's conda environments In this section we use conda to install the versions of the language interpreters and third-party libraries required by the package's diagnostics. -- First, determine the location of your conda/micromamba installation by running :console:`% conda info --base` or :console:`% micromamba info --base` -as the user who will be using the package. This path will be referred to as <*CONDA_ROOT*> below. +- First, determine the location of your conda/micromamba installation by running :console:`% conda info --base` or :console:`% micromamba info` +as the user who will be using the package. This path will be referred to as <*CONDA_ROOT*> or <*MICROMAMBA_ROOT*> below. -- If you don't have write access to <*CONDA_ROOT*> +- If you don't have write access to <*CONDA_ROOT*>/<*MICROMAMBA_ROOT*> (for example, if conda has been installed for all users of a multi-user system), you will need to tell conda to install its files in a different, writable location. You can also choose to do this out of convenience, e.g. to keep all files and programs used by the MDTF package together in the ``mdtf`` directory for organizational purposes. This location will be referred to as <*CONDA_ENV_DIR*> below. +To display information about all of the options in the conda_env_setup.sh and +micromamba_env_setup.sh environment installation scripts, run + +.. code-block:: console + + % cd + % ./src/conda/conda_env_setup.sh [-h|--help] + % ./src/conda/micromamba_env_setup.sh [-h|--help] + - Install all the package's conda environments with anaconda/miniconda by running .. code-block:: console @@ -232,9 +241,10 @@ in the ``mdtf`` directory for organizational purposes. This location will be ref .. code-block:: console % cd - % ./src/conda/conda_env_setup.sh --all --micromamba_root --env_dir - <*MICROMAMBA_ROOT*> is the path to the micromamba executable on your system (e.g., /home/${USER/.local/bin) + % ./src/conda/micromamba_env_setup.sh --all --micromamba_root --micromamba_exe --env_dir + <*MICROMAMBA_ROOT*> is the path to the micromamba installation on your system (e.g., /home/${USER}/micromamba) + <*MICROMAMBA_EXE*> is the path to the micromamba executable on your system (e.g., /home/${USER}/.local/bin/micromamba) .. note:: Micromamba is required to install the conda environments on machines with Apple M-series chips. @@ -245,8 +255,8 @@ in the ``mdtf`` directory for organizational purposes. This location will be ref .. code-block:: console % cd - % ./src/conda/conda_env_setup.sh -e base --micromamba_root --env_dir - % ./src/conda/conda_env_setup.sh -e python3_ ase --micromamba_root --env_dir + % ./src/conda/micromamba_env_setup.sh -e base --micromamba_root --micromamba_exe --env_dir + % ./src/conda/micromamba_env_setup.sh -e python3_base --micromamba_root --micromamba_exe --env_dir .. note:: diff --git a/mdtf_framework.py b/mdtf_framework.py index 95286744c..c6411b599 100755 --- a/mdtf_framework.py +++ b/mdtf_framework.py @@ -9,8 +9,8 @@ import sys # do version check before anything else -if sys.version_info.major != 3 or sys.version_info.minor < 11: - sys.exit("ERROR: The MDTF package requires python >= 3.11. Please check " +if sys.version_info.major != 3 or sys.version_info.minor < 10: + sys.exit("ERROR: The MDTF package requires python >= 3.10. Please check " "which version of python is on your $PATH (e.g. with `which python`.)\n" f"Attempted to run with following python version:\n{sys.version}") # passed; continue with imports diff --git a/sites/NOAA_GFDL/default_gfdl.jsonc b/sites/NOAA_GFDL/default_gfdl.jsonc index c0549dd80..c1a390f00 100644 --- a/sites/NOAA_GFDL/default_gfdl.jsonc +++ b/sites/NOAA_GFDL/default_gfdl.jsonc @@ -101,15 +101,17 @@ // Settings affecting the runtime environment of the PODs. "environment_manager": "conda", - // Path to the Anaconda installation. Only used if environment_manager='Conda'. - // Set equal to "" to use conda from your system's $PATH. + // Path to the Anaconda, miniconda, or micromamba installation. + // Only used if environment_manager='Conda'. "conda_root": "/home/oar.gfdl.mdtf/miniconda3", // Root directory for Anaconda environment installs. Only used if - // environment_manager = 'Conda'. Set equal to '' to install in your system's - // default location. + // environment_manager = 'Conda'. "conda_env_root": "/home/oar.gfdl.mdtf/miniconda3/envs", + // Path to micromamba executable. Required if using micromamba to manage environments + "micromamba_exe": "", + // Root directory for python virtual environments. Only used if // environment_manager = 'Virtualenv'. Set equal to '' to install in your // system's default location. diff --git a/sites/NOAA_GFDL/gfdl_multirun_config_template.jsonc b/sites/NOAA_GFDL/gfdl_multirun_config_template.jsonc index e79f3fe88..c96a73953 100644 --- a/sites/NOAA_GFDL/gfdl_multirun_config_template.jsonc +++ b/sites/NOAA_GFDL/gfdl_multirun_config_template.jsonc @@ -117,6 +117,9 @@ // default location. "conda_env_root": "/home/oar.gfdl.mdtf/miniconda3/envs", + // Path to micromamba executable. Required if using micromamba to manage environments + "micromamba_exe": "", + // Root directory for python virtual environments. Only used if // environment_manager = 'Virtualenv'. Set equal to '' to install in your // system's default location. diff --git a/src/cli_plugins.jsonc b/src/cli_plugins.jsonc index 43fa184cc..b6bf2a364 100644 --- a/src/cli_plugins.jsonc +++ b/src/cli_plugins.jsonc @@ -106,6 +106,11 @@ "default" : "", "metavar" : "", "action": "PathAction" + },{ + "name": "micromamba_exe", + "help": "Path to micromamba executable", + "default" : "", + "metavar" : "" } ] } diff --git a/src/conda/conda_env_setup.sh b/src/conda/conda_env_setup.sh index 2c23a2663..67b0a2239 100755 --- a/src/conda/conda_env_setup.sh +++ b/src/conda/conda_env_setup.sh @@ -6,7 +6,22 @@ set -Eeo pipefail # Enable extended globbing, see # https://www.gnu.org/software/bash/manual/bashref.html#Pattern-Matching shopt -s extglob - +######################### +# The command line help +# Everything is stackoverflow: https://stackoverflow.com/questions/5474732/how-can-i-add-a-help-method-to-a-shell-script +######################### +display_help() { + echo "Usage: ./src/conda/conda_env_setup [option 1, ... option n]" >&2 + echo "" + echo " -a, --all build all conda environments in src/conda with 'env' prefix" + echo " -e, --env [base | python3_base | R_base | NCL_base] build specific environment defined in src/conda/env_[name]_base.yml " + echo " -cr, --conda_root root path to anaconda or miniconda directory" + echo " -d, --env_dir directory path where conda environments will be installed" + echo " --wrapper_only do not change conda enviroments; only build the mdtf wrapper" + echo "" + # echo some stuff here for the -a or --add-options + exit 1 +} # get directory this script is located in, resolving any # symlinks/aliases (https://stackoverflow.com/a/246128) _source="${BASH_SOURCE[0]}" @@ -24,14 +39,18 @@ script_dir="$( cd -P "$( dirname "$_source" )" >/dev/null 2>&1 && pwd )" repo_dir="$( cd -P "$script_dir" >/dev/null 2>&1 && cd ../../ && pwd )" pushd "$PWD" > /dev/null -# parse arguments manually +# parse aruments manually _MDTF_CONDA_ROOT="" -_MDTF_MICROMAMBA_ROOT="" _CONDA_ENV_ROOT="" make_envs="true" env_glob="" while (( "$#" )); do case "$1" in + # call the help function + -h | --help) + display_help + exit 0 + ;; -a|--all) # install all envs except dev environment env_glob="env_!(dev).yml" @@ -65,10 +84,9 @@ while (( "$#" )); do fi cd "$2" export _CONDA_ENV_ROOT="$PWD" - echo "$_CONDA_ENV_ROOT" shift 2 ;; - -cr|--conda_root) + -c|--conda_root) # manually specify path to conda installation; resolve path first cd "$repo_dir" if [ ! -d "$2" ]; then @@ -79,17 +97,6 @@ while (( "$#" )); do export _MDTF_CONDA_ROOT="$PWD" shift 2 ;; - -mr|--micromamba_root) - # manually specify path to micromamba installation; resolve path first - cd "$repo_dir" - if [ ! -d "$2" ]; then - echo "ERROR: can't find micromamba directory $2" - exit 1 - fi - cd "$2" - export _MDTF_MICROMAMBA_ROOT="$PWD" - shift 2 - ;; --wrapper-only) # Don't change conda envs; only update wrapper shell script echo "Updating wrapper script and leaving envs unchanged." @@ -109,18 +116,12 @@ popd > /dev/null # restore CWD # setup conda for non-interactive shell # NB: 'conda' isn't an executable; it's created as a shell alias. This is why we # invoke it as 'conda' below, instead of the absolute path in $CONDA_EXE. -if [[ -z "$_MDTF_CONDA_ROOT" && -z "$_MDTF_MICROMAMBA_ROOT" ]]; then +if [ -z "$_MDTF_CONDA_ROOT" ]; then set -- # clear cmd line - echo "calling conda_init.sh" . "${script_dir}/conda_init.sh" -v -elif [[ -z "$_MDTF_MICROMAMBA_ROOT" && -n "$_MDTF_CONDA_ROOT" ]]; then +else # pass conda installation dir to setup script - echo "calling conda_init.sh on conda_root" . "${script_dir}/conda_init.sh" -v "$_MDTF_CONDA_ROOT" -elif [[ -n "$_MDTF_MICROMAMBA_ROOT" && -z "$_MDTF_CONDA_ROOT" ]]; then - # pass conda installation dir to setup script - echo "calling conda_init.sh on $_MDTF_MICROMAMBA_ROOT" - . "${script_dir}/conda_init.sh" -v "$_MDTF_MICROMAMBA_ROOT" fi if [ "$make_envs" = "true" ]; then @@ -136,39 +137,20 @@ if [ "$make_envs" = "true" ]; then # install envs with mamba (https://github.com/mamba-org/mamba) for # performance reasons; need to find mamba executable, or install it if not # present - if [ -n "$_MDTF_CONDA_ROOT" ]; then - _INSTALL_EXE=$( command -v mamba ) || true - mamba_temp="false" - if [[ ! -x "$_INSTALL_EXE" ]]; then - echo "Couldn't find mamba executable; installing in temp environment." - mamba_temp="true" - conda create --force -qy -n _MDTF_install_temp - conda install -qy mamba -n _MDTF_install_temp -c conda-forge - # still no idea why this works but "conda activate" doesn't - conda activate _MDTF_install_temp - _INSTALL_EXE=$( command -v mamba ) || true - fi - if [[ ! -x "$_INSTALL_EXE" ]]; then - echo "Mamba installation failed." - exit 1 - fi - elif [ -n "$_MDTF_MICROMAMBA_ROOT" ]; then - # not set, create conda env without --prefix - echo "Installing envs with micromamba" - #. "${_CONDA_ROOT}/micromamba/etc/profile.d/micromamba.sh" - if [ -f "$HOME/.bash_profile" ]; then - source "$HOME/.bash_profile" - elif [ -f "$HOME/.bashrc" ]; then - source "$HOME/.bashrc" - else - echo "adding ${_CONDA_ROOT}/bin to \$PATH" - export PATH="${_CONDA_ROOT}/bin:$PATH" - fi - _INSTALL_EXE=$( command -v micromamba ) || true - if [[ -z "$_INSTALL_EXE" ]]; then - echo "Error: micromamba not found." - exit 1 - fi + _INSTALL_EXE=$( command -v mamba ) || true + mamba_temp="false" + if [[ ! -x "$_INSTALL_EXE" ]]; then + echo "Couldn't find mamba executable; installing in temp environment." + mamba_temp="true" + conda create --force -qy -n _MDTF_install_temp + conda install -qy mamba -n _MDTF_install_temp -c conda-forge + # still no idea why this works but "conda activate" doesn't + conda activate _MDTF_install_temp + _INSTALL_EXE=$( command -v mamba ) || true + fi + if [[ ! -x "$_INSTALL_EXE" ]]; then + echo "Mamba installation failed." + exit 1 fi # create all envs in a loop @@ -177,26 +159,15 @@ if [ "$make_envs" = "true" ]; then [ -e "$env_file" ] || continue # catch the case where nothing matches # get env name from reading "name:" attribute of yaml file env_name=$( sed -n "s/^[[:space:]]*name:[[:space:]]*\([[:alnum:]_\-]*\)[[:space:]]*/\1/p" "$env_file" ) - if [[ -z "$_CONDA_ENV_ROOT" && -z "$_MDTF_MICROMAMBA_ROOT" ]]; then + if [ -z "$_CONDA_ENV_ROOT" ]; then # need to set manually, otherwise mamba will install in a subdir # of its env's directory - conda_prefix="${_CONDA_ROOT}/envs/${env_name}" - echo "Creating conda env ${env_name} in ${conda_prefix}..." - "$_INSTALL_EXE" env create --force -q -p="$conda_prefix" -f="$env_file" - elif [ -n "$_MDTF_MICROMAMBA_ROOT" ]; then - if [ -n "$_CONDA_ENV_ROOT" ]; then - conda_prefix="${_CONDA_ENV_ROOT}/${env_name}" - else - conda_prefix="${_MDTF_MICROMAMBA_ROOT}/envs/${env_name}" - fi - echo "Creating conda env ${env_name} in ${conda_prefix}..." - "$_INSTALL_EXE" create -q -y -p "$conda_prefix" -f "$env_file" + conda_prefix="${_CONDA_ROOT}/envs/${env_name}" else - conda_prefix="${_CONDA_ENV_ROOT}/${env_name}" - echo "Creating conda env ${env_name} in ${conda_prefix}..." - "$_INSTALL_EXE" env create --force -q -p="$conda_prefix" -f="$env_file" + conda_prefix="${_CONDA_ENV_ROOT}/${env_name}" fi - + echo "Creating conda env ${env_name} in ${conda_prefix}..." + "$_INSTALL_EXE" env create --force -q -p="$conda_prefix" -f="$env_file" echo "... conda env ${env_name} created." done "$_INSTALL_EXE" clean -aqy @@ -209,7 +180,6 @@ if [ "$make_envs" = "true" ]; then fi # create script wrapper to activate base environment -echo "creating mdtf wrapper" _CONDA_WRAPPER="${repo_dir}/mdtf" if [ -e "$_CONDA_WRAPPER" ]; then rm -f "$_CONDA_WRAPPER" @@ -218,14 +188,12 @@ echo '#!/usr/bin/env bash' > "$_CONDA_WRAPPER" echo "# This wrapper script is generated by conda_env_setup.sh." >> "$_CONDA_WRAPPER" echo "_mdtf=\"${repo_dir}\"" >> "$_CONDA_WRAPPER" echo "source \"\${_mdtf}/src/conda/conda_init.sh\" -q \"${_CONDA_ROOT}\"" >> "$_CONDA_WRAPPER" -if [[ -z "$_CONDA_ENV_ROOT" && -z "$_MDTF_MICROMAMBA_ROOT" ]]; then +if [ -z "$_CONDA_ENV_ROOT" ]; then echo "conda activate _MDTF_base" >> "$_CONDA_WRAPPER" -elif [[ -n "$_MDTF_MICROMAMBA_ROOT" ]]; then - echo "micromamba activate _MDTF_base" >> "$_CONDA_WRAPPER" else echo "conda activate ${_CONDA_ENV_ROOT}/_MDTF_base" >> "$_CONDA_WRAPPER" fi echo "\"\${_mdtf}/mdtf_framework.py\" \"\$@\"" >> "$_CONDA_WRAPPER" echo "exit \$?" >> "$_CONDA_WRAPPER" chmod +x "$_CONDA_WRAPPER" -echo "Created MDTF wrapper script at ${_CONDA_WRAPPER}" +echo "Created MDTF wrapper script at ${_CONDA_WRAPPER}" \ No newline at end of file diff --git a/src/conda/conda_init.sh b/src/conda/conda_init.sh index 610167db8..658a795ef 100755 --- a/src/conda/conda_init.sh +++ b/src/conda/conda_init.sh @@ -93,12 +93,6 @@ if [[ -d "$_CONDA_ROOT" && ! -x "$CONDA_EXE" ]]; then elif [ -x "${_CONDA_ROOT}/condabin/conda" ]; then CONDA_EXE="${_CONDA_ROOT}/condabin/conda" if [ $_v -eq 2 ]; then echo "Found CONDA_EXE"; fi - elif [ -x "${_CONDA_ROOT}/bin/micromamba" ]; then - CONDA_EXE="${_CONDA_ROOT}/bin/micromamba" - if [ $_v -eq 2 ]; then echo "Found CONDA_EXE--using micromamba"; fi - elif [ -x "${_CONDA_ROOT}/micromamba" ]; then - CONDA_EXE="${_CONDA_ROOT}/micromamba" - if [ $_v -eq 2 ]; then echo "Found CONDA_EXE--using micromamba"; fi fi fi # found exe but not root @@ -124,10 +118,7 @@ else if [ ! -d "$_CONDA_ROOT" ]; then echo "ERROR: search for conda base dir failed (${_CONDA_ROOT})" 1>&2 fi - - conda_check=$( $CONDA_EXE info ) - - if [ -z "$conda_check" ]; then + if [ ! -x "$CONDA_EXE" ]; then echo "ERROR: search for conda executable failed (${CONDA_EXE})" 1>&2 fi exit 1 @@ -137,26 +128,14 @@ fi # https://github.com/conda/conda/issues/9392#issuecomment-617345019 unset CONDA_SHLVL # finally run conda's init script - -if [ -x "eval conda info" ]; then - __conda_setup="$( $CONDA_EXE 'shell.bash' 'hook' 2> /dev/null )" -fi - +__conda_setup="$( $CONDA_EXE 'shell.bash' 'hook' 2> /dev/null )" if [ $? -eq 0 ]; then eval "$__conda_setup" else if [ -f "${_CONDA_ROOT}/etc/profile.d/conda.sh" ]; then - echo "calling conda.sh" . "${_CONDA_ROOT}/etc/profile.d/conda.sh" - - elif [ -f "${_CONDA_ROOT}/etc/profile.d/micromamba.sh" ]; then - echo "calling micromamba.sh" - . "${_CONDA_ROOT}/etc/profile.d/micromamba.sh" else - echo "adding ${_CONDA_ROOT}/bin to \$PATH" export PATH="${_CONDA_ROOT}/bin:$PATH" - fi fi unset __conda_setup - diff --git a/src/conda/env_base.yml b/src/conda/env_base.yml index 77016aa14..f753be183 100644 --- a/src/conda/env_base.yml +++ b/src/conda/env_base.yml @@ -28,3 +28,4 @@ dependencies: - subprocess32=3.5.4 - pyyaml=6.0.1 - click=8.1.7 +- ghostscript=10.02.1 diff --git a/src/conda/micromamba_env_setup.sh b/src/conda/micromamba_env_setup.sh new file mode 100755 index 000000000..0cac2cd5e --- /dev/null +++ b/src/conda/micromamba_env_setup.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# Driver script to create Anaconda environments for MDTF. +# Require bash due to lingering conda compatibility issues. + +set -Eeo pipefail +# Enable extended globbing, see +# https://www.gnu.org/software/bash/manual/bashref.html#Pattern-Matching +shopt -s extglob +######################### +# The command line help +# Everything is stackoverflow: https://stackoverflow.com/questions/5474732/how-can-i-add-a-help-method-to-a-shell-script +######################### +display_help() { + echo "Usage: ./src/conda/conda_env_setup [option 1, ... option n]" >&2 + echo "" + echo " -a, --all build all conda environments in src/conda with 'env' prefix" + echo " -e, --env [base | python3_base | R_base | NCL_base] build specific environment defined in src/conda/env_[name]_base.yml " + echo " -mr, --micromamba_root root path to micromamba directory" + echo " --micromamba_exe full path to micromamba executable on your system" + echo " -d, --env_dir directory path where conda environments will be installed" + echo " --wrapper_only do not change conda enviroments; only build the mdtf wrapper" + echo "" + # echo some stuff here for the -a or --add-options + exit 1 +} + +# get directory this script is located in, resolving any +# symlinks/aliases (https://stackoverflow.com/a/246128) +_source="${BASH_SOURCE[0]}" +while [ -h "$_source" ]; do # resolve $_source until the file is no longer a symlink + script_dir="$( cd -P "$( dirname "$_source" )" >/dev/null 2>&1 && pwd )" + _source="$( readlink "$_source" )" + # if $_source was a relative symlink, we need to resolve it relative to the + # path where the symlink file was located + [[ $_source != /* ]] && _source="$script_dir/$_source" +done +script_dir="$( cd -P "$( dirname "$_source" )" >/dev/null 2>&1 && pwd )" + +# relative paths resolved relative to repo directory, which is grandparent +# of dir this script is in +repo_dir="$( cd -P "$script_dir" >/dev/null 2>&1 && cd ../../ && pwd )" + +pushd "$PWD" > /dev/null +# parse arguments manually +_MDTF_CONDA_ROOT="" +_MDTF_MICROMAMBA_ROOT="" +_MDTF_MICROMAMBA_EXE="" +_CONDA_ENV_ROOT="" +make_envs="true" +env_glob="" +while (( "$#" )); do + case "$1" in + -h | --help) + display_help # call the help function + exit 0 + ;; + -a|--all) + # install all envs except dev environment + env_glob="env_!(dev).yml" + shift 1 + ;; + -e|--env) + # specify one env by name + env_glob="env_${2}.yml" + if [ ! -f "${script_dir}/${env_glob}" ]; then + echo "ERROR: ${script_dir}/${env_glob} not found." + exit 1 + fi + shift 2 + ;; + --dev-only) + # dev and base only (for Travis CI/ auto testing) + env_glob="env_@(base|dev).yml" + shift 1 + ;; + --all-dev) + # all envs, including dev + env_glob="env_*.yml" + shift 1 + ;; + -d|--env_dir) + # specify install destination; resolve path first + cd "$repo_dir" + if [ ! -d "$2" ]; then + echo "Creating directory $2" + mkdir -p "$2" + fi + cd "$2" + export _CONDA_ENV_ROOT="$PWD" + echo "$_CONDA_ENV_ROOT" + shift 2 + ;; + -mr|--micromamba_root) + # manually specify path to micromamba installation; resolve path first + cd "$repo_dir" + if [ ! -d "$2" ]; then + echo "ERROR: can't find micromamba directory $2" + exit 1 + fi + cd "$2" + export _MDTF_MICROMAMBA_ROOT="$PWD" + shift 2 + ;; + --micromamba_exe) + # path to micromamba executable + if [ ! -x "$2" ]; then + echo "ERROR: can't find micromamba executable $2" + exit 1 + fi + export _MDTF_MICROMAMBA_EXE="$2" + shift 2 + ;; + --wrapper-only) + # Don't change conda envs; only update wrapper shell script + echo "Updating wrapper script and leaving envs unchanged." + make_envs="false" + shift 1 + ;; + -?*) + echo "$0: Unknown option (ignored): $1\n" >&2 + shift 1 + ;; + *) # Default case: No more options, so break out of the loop. + break + esac +done +popd > /dev/null # restore CWD + +# setup conda for non-interactive shell +# NB: 'conda' isn't an executable; it's created as a shell alias. This is why we +# invoke it as 'conda' below, instead of the absolute path in $CONDA_EXE. +if [[ -z "$_MDTF_MICROMAMBA_ROOT" ]]; then + set -- # clear cmd line + echo "calling micromamba_init.sh" + . "${script_dir}/micromamba_init.sh" -v +elif [ -n "$_MDTF_MICROMAMBA_ROOT" ]; then + if [ -n "$_MDTF_MICROMAMBA_EXE" ]; then + # pass micromamba installation dir and executable path to setup script + echo "calling micromamba_init.sh on $_MDTF_MICROMAMBA_ROOT and $_MDTF_MICROMAMBA_EXE" + . "${script_dir}/micromamba_init.sh" -v --micromamba_root "$_MDTF_MICROMAMBA_ROOT" --micromamba_exe "$_MDTF_MICROMAMBA_EXE" + else + . "${script_dir}/micromamba_init.sh" -v --micromamba_root "$_MDTF_MICROMAMBA_ROOT" + fi +fi + +echo "MICROMAMBA_ROOT is $_CONDA_ROOT" +echo "MICROMAMBA_EXE is $CONDA_EXE" + +if [ "$make_envs" = "true" ]; then + if [ -z "$_CONDA_ENV_ROOT" ]; then + # not set, create conda env without --prefix + echo "Installing envs into system Anaconda" + else + # set, create and change conda envs using --prefix + echo "Installing envs into $_CONDA_ENV_ROOT" + echo "To use envs interactively, run \"conda config --append envs_dirs $_CONDA_ENV_ROOT\"" + fi + + if [ -n "$_MDTF_MICROMAMBA_ROOT" ]; then + _INSTALL_EXE=$( command -v micromamba ) || true + if [[ -z "$_INSTALL_EXE" ]]; then + echo "Error: micromamba not found." + exit 1 + fi + fi + + # create all envs in a loop + "$_INSTALL_EXE" clean -qi + for env_file in "${script_dir}/"${env_glob}; do + [ -e "$env_file" ] || continue # catch the case where nothing matches + # get env name from reading "name:" attribute of yaml file + env_name=$( sed -n "s/^[[:space:]]*name:[[:space:]]*\([[:alnum:]_\-]*\)[[:space:]]*/\1/p" "$env_file" ) + if [ -n "$_CONDA_ENV_ROOT" ]; then + conda_prefix="${_CONDA_ENV_ROOT}/${env_name}" + else + conda_prefix="${_MDTF_MICROMAMBA_ROOT}/envs/${env_name}" + fi + + echo "Creating conda env ${env_name} in ${conda_prefix}..." + "$_INSTALL_EXE" create -q -y -p "$conda_prefix" -f "$env_file" + echo "... conda env ${env_name} created." + done + "$_INSTALL_EXE" clean -aqy +fi + +# create script wrapper to activate base environment +echo "creating mdtf wrapper" +_CONDA_WRAPPER="${repo_dir}/mdtf" +if [ -e "$_CONDA_WRAPPER" ]; then + rm -f "$_CONDA_WRAPPER" +fi +echo '#!/usr/bin/env bash' > "$_CONDA_WRAPPER" +echo "# This wrapper script is generated by conda_env_setup.sh." >> "$_CONDA_WRAPPER" +echo "_mdtf=\"${repo_dir}\"" >> "$_CONDA_WRAPPER" + +if [ -n "$_MDTF_MICROMAMBA_EXE" ]; then + echo "source \"\${_mdtf}/src/conda/micromamba_init.sh\" -q --micromamba_root \"${_CONDA_ROOT}\" --micromamba_exe \"${_MDTF_MICROMAMBA_EXE}\"" >> "$_CONDA_WRAPPER" +else + echo "source \"\${_mdtf}/src/conda/micromamba_init.sh\" -q --micromamba_root \"${_CONDA_ROOT}\"" >> "$_CONDA_WRAPPER" +fi + +echo "micromamba activate _MDTF_base" >> "$_CONDA_WRAPPER" + +echo "\"\${_mdtf}/mdtf_framework.py\" \"\$@\"" >> "$_CONDA_WRAPPER" +echo "exit \$?" >> "$_CONDA_WRAPPER" +chmod +x "$_CONDA_WRAPPER" +echo "Created MDTF wrapper script at ${_CONDA_WRAPPER}" diff --git a/src/conda/micromamba_init.sh b/src/conda/micromamba_init.sh new file mode 100755 index 000000000..ddc8f8649 --- /dev/null +++ b/src/conda/micromamba_init.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +# This is a workaround to permit switching conda environments in a +# non-interactive shell. +# The script is what's placed in ~/.bashrc by 'conda init bash'; +# this doesn't get sourced by bash in non-interactive mode so we have to +# do it manually. See https://github.com/conda/conda/issues/7980 . + +# NOTE this has only been tested with conda 4.7.10 and later; I know earlier +# versions had things in different places. + +# parse arguments manually +_TEMP_CONDA_ROOT="" +_TEMP_CONDA_EXE="" +_TEMP_CONDA_ENV_DIR="" +_TEMP_MICROMAMBA_EXE="" +_v=1 +while (( "$#" )); do + case "$1" in + -v) + _v=2 # verbose output for debugging + shift 1 + ;; + -q) + _v=0 # suppress output + shift 1 + ;; + --micromamba_root) + if [ ! -d "$2" ]; then + echo "ERROR: \"$2\" not a directory" 1>&2 + exit 1 + fi + _TEMP_CONDA_ROOT="$2" + shift 2 + ;; + --micromamba_exe) + # path to micromamba executable + if [ ! -x "$2" ]; then + echo "ERROR: can't find micromamba executable $2" + exit 1 + fi + export _TEMP_MICROMAMBA_EXE="$2" + shift 2 + ;; + *) # Default case: No more options, so break out of the loop. + break + esac +done + +if [ -x "$_TEMP_MICROMAMBA_EXE" ]; then + echo "CONDA_EXE=$_TEMP_MICROMAMBA_EXE" + CONDA_EXE="$_TEMP_MICROMAMBA_EXE" +fi + +# if we got _TEMP_CONDA_ROOT from command line, see if that works +if [ -d "$_TEMP_CONDA_ROOT" ]; then + # let command line value override pre-existing _CONDA_ROOT, in case user + # is specifying personal vs. site installation of conda + if [[ $_v -eq 2 && -d "$_CONDA_ROOT" ]]; then + echo "WARNING: overriding ${_CONDA_ROOT} with ${_TEMP_CONDA_ROOT}" 1>&2 + fi + _CONDA_ROOT="$_TEMP_CONDA_ROOT" + + if [ $_v -eq 2 ]; then echo "CONDA_ROOT set from command line"; fi +fi +# if not, maybe we were run from an interactive shell and inherited the info +if [ ! -d "$_CONDA_ROOT" ]; then + if [ -x "$CONDA_EXE" ]; then + if [ $_v -eq 2 ]; then echo "CONDA_EXE set from environment"; fi + _TEMP_CONDA_ROOT="$( "$CONDA_EXE" info --base 2> /dev/null )" + else + _TEMP_CONDA_ROOT="$( conda info --base 2> /dev/null )" + fi + if [ -d "$_TEMP_CONDA_ROOT" ]; then + _CONDA_ROOT="$_TEMP_CONDA_ROOT" + if [ $_v -eq 2 ]; then echo "CONDA_ROOT set from environment"; fi + fi +fi +# if not, run user's shell in interactive mode. Subshell output could have +# arbitrary text output in it, since user's init scripts may be setting prompt +# and generating output in any number of ways. We try to extract the paths by +# delimiting them with (hopefully uncommon) vertical tab characters (\v) and +# using awk to extract whatever text is found between those two field separators. +if [ ! -d "$_CONDA_ROOT" ]; then + if [ $_v -eq 2 ]; then echo "Setting conda from $SHELL -i"; fi + _TEMP_CONDA_ROOT=$( "$SHELL" -i -c "_temp=\$( conda info --base ) && echo \"\v\${_temp}\v\"" | awk 'BEGIN { FS = "\v" } ; { print $2 }' ) + if [ $_v -eq 2 ]; then echo "Received MICROMAMBA_ROOT=\"${_TEMP_CONDA_ROOT}\""; fi + if [[ -d "$_TEMP_CONDA_ROOT" ]]; then + _CONDA_ROOT="$_TEMP_CONDA_ROOT" + if [ $_v -eq 2 ]; then echo "Found MICROMAMBA_ROOT"; fi + fi + _TEMP_CONDA_EXE="$( "$SHELL" -i -c "echo \"\v\${CONDA_EXE}\v\"" | awk 'BEGIN { FS = "\v" } ; { print $2 }' )" + if [ $_v -eq 2 ]; then echo "Received MICROMAMBA_EXE=\"${_TEMP_CONDA_EXE}\""; fi + if [[ ! -x "$CONDA_EXE" && -x "$_TEMP_CONDA_EXE" ]]; then + CONDA_EXE="$_TEMP_CONDA_EXE" + if [ $_v -eq 2 ]; then echo "Found CONDA_EXE"; fi + fi +fi +# found root but not exe +if [[ -d "$_CONDA_ROOT" && ! -x "$CONDA_EXE" ]]; then + if [ $_v -eq 2 ]; then echo "Looking for micromamba executable in ${_CONDA_ROOT}"; fi + if [ -x "${_CONDA_ROOT}/bin/micromamba" ]; then + CONDA_EXE="${_CONDA_ROOT}/bin/micromamba" + if [ $_v -eq 2 ]; then echo "Found CONDA_EXE--using micromamba"; fi + elif [ -x "${_CONDA_ROOT}/micromamba" ]; then + CONDA_EXE="${_CONDA_ROOT}/micromamba" + if [ $_v -eq 2 ]; then echo "Found CONDA_EXE--using micromamba"; fi + fi +fi +# found exe but not root +if [[ -x "$CONDA_EXE" && ! -d "$_CONDA_ROOT" ]]; then + if [ $_v -eq 2 ]; then echo "Running $CONDA_EXE to find conda root"; fi + _TEMP_CONDA_ROOT="$( "$CONDA_EXE" info --base 2> /dev/null )" + if [ -d "$_TEMP_CONDA_ROOT" ]; then + _CONDA_ROOT="$_TEMP_CONDA_ROOT" + if [ $_v -eq 2 ]; then echo "Found CONDA_ROOT"; fi + fi +fi + +if [[ -x "$CONDA_EXE" && -d "$_CONDA_ROOT" ]]; then + if [ $_v -ne 0 ]; then + # Conda env manager reads this output + echo "_CONDA_EXE=${CONDA_EXE}" + echo "_CONDA_ROOT=${_CONDA_ROOT}" + fi + # in case these weren't exported already + export _CONDA_ROOT="$_CONDA_ROOT" + export CONDA_EXE="$CONDA_EXE" +else + if [ ! -d "$_CONDA_ROOT" ]; then + echo "ERROR: search for conda base dir failed (${_CONDA_ROOT})" 1>&2 + fi + + conda_check=$( $CONDA_EXE info ) + + if [ -z "$conda_check" ]; then + echo "ERROR: search for micromamba executable failed (${CONDA_EXE})" 1>&2 + fi + exit 1 +fi + +# workaround for $PATH being set incorrectly +# https://github.com/conda/conda/issues/9392#issuecomment-617345019 +unset CONDA_SHLVL +# finally run micromamba's init script + +conda_check=$( $CONDA_EXE info ) +__conda_setup="" +if [ -n "$conda_check" ]; then + if [ -n "$_TEMP_MICROMAMBA_EXE" ]; then + __conda_setup="$( "$CONDA_EXE" shell hook --shell bash --root-prefix "${_CONDA_ROOT}" 2> /dev/null )" + fi +fi + +if [[ $? -eq 0 && -n "$__conda_setup" ]]; then + eval "$__conda_setup" +else + if [ -n "$_TEMP_MICROMAMBA_EXE" ]; then + alias micromamba="$CONDA_EXE" + else + export PATH="${_CONDA_ROOT}/bin:$PATH" + fi +fi +unset __conda_setup + diff --git a/src/core.py b/src/core.py index 750f5fd92..9ad09df02 100644 --- a/src/core.py +++ b/src/core.py @@ -235,6 +235,10 @@ def __init__(self, cli_obj=None, env_vars=None, unittest=False): # in its definition in the .jsonc file cli_paths = [act.dest for act in cli_obj.iter_actions() if isinstance(act, cli.PathAction)] + if 'micromamba_exe' in d: + if os.path.exists(d['micromamba_exe']): + cli_paths.append('micromamba_exe') + if not cli_paths: _log.warning("Didn't get list of paths from CLI.") for key in cli_paths: diff --git a/src/default_tests.jsonc b/src/default_tests.jsonc index a3a4fa228..dbe8027d9 100644 --- a/src/default_tests.jsonc +++ b/src/default_tests.jsonc @@ -81,11 +81,14 @@ // put in a subdirectory of this directory. "OUTPUT_DIR": "../wkdir", - // Location of the Anaconda/miniconda or micromamba binary installation to use for managing + // Location of the Anaconda/miniconda or micromamba installation to use for managing // dependencies (path returned by running `[conda | micromamba] info`.) If empty, // framework will attempt to determine location of system's conda installation. "conda_root": "", + // Location of micromamba executable if using micromamba + "micromamba_exe":"", + // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left // blank, the framework will look for its environments in the system default diff --git a/src/environment_manager.py b/src/environment_manager.py index df3b7ae33..41232ecb3 100644 --- a/src/environment_manager.py +++ b/src/environment_manager.py @@ -238,9 +238,14 @@ def __init__(self, log=_log): # find conda executable # conda_init for bash defines conda as a shell function; will get error # if we try to call the conda executable directly + if any(paths.get('micromamba_exe','')): + cmd = f"{self.conda_dir}/micromamba_init.sh --micromamba_root {paths.get('conda_root','')} --micromamba_exe {paths.get('micromamba_exe','')}" + else: + cmd = f"{self.conda_dir}/conda_init.sh {paths.get('conda_root','')}" + try: conda_info = util.run_shell_command( - f"{self.conda_dir}/conda_init.sh {paths.get('conda_root','')}", + cmd, log=self.log ) for line in conda_info: @@ -331,25 +336,9 @@ def activate_env_commands(self, env_name): # conda_init for bash defines conda as a shell function; will get error # if we try to call the conda executable directly conda_prefix = os.path.join(self.conda_env_root, env_name) - env_config_file = "" if os.path.split(self.conda_exe)[-1] == 'micromamba': - # micromamba appends env info to the shell config file - # that needs to be sourced in the sub-shell for each POD process - home_dir = os.path.expanduser('~') - if os.path.isfile(os.path.join(home_dir, '.bashrc')): - env_config_file = os.path.join(home_dir, '.bashrc') - elif os.path.isfile(os.path.join(home_dir, '.bash_profile')): - env_config_file = os.path.join(home_dir, '.bash_profile') - elif os.path.isfile(os.path.join(home_dir, '.cshrc')): - env_config_file = os.path.join(home_dir, '.cshrc') - elif os.path.isfile(os.path.join(home_dir, '.tcshrc')): - env_config_file = os.path.join(home_dir, '.tcshrc') - elif os.path.isfile(os.path.join(home_dir, '.zshrc')): - env_config_file = os.path.join(home_dir, '.zshrc') return [ - f'source {self.conda_dir}/conda_init.sh {self.conda_root}', - f'source {env_config_file}', - f'eval "$(micromamba shell hook --shell bash)"', + f'source {self.conda_dir}/micromamba_init.sh --micromamba_exe {self.conda_exe} --conda_root {self.conda_root}', f'micromamba activate {conda_prefix}' ] else: diff --git a/src/output_manager.py b/src/output_manager.py index 5c5da226c..0028421d3 100644 --- a/src/output_manager.py +++ b/src/output_manager.py @@ -139,7 +139,7 @@ def make_pod_html(self): overwrite=True ) - def convert_pod_figures(self, src_subdir, dest_subdir): + def convert_pod_figures(self, src_subdir: str, dest_subdir: str): """Convert all vector graphics in ``$POD_WK_DIR/`` *src\_subdir* to .png files using `ghostscript `__ (included in the \_MDTF\_base conda environment). @@ -160,7 +160,7 @@ def convert_pod_figures(self, src_subdir, dest_subdir): # Flags to pass to ghostscript for PS -> PNG conversion (in particular # bitmap resolution.) eps_convert_flags = ("-dSAFER -dBATCH -dNOPAUSE -dEPSCrop -r150 " - "-sDEVICE=png16m -dTextAlphaBits=4 -dGraphicsAlphaBits=4") + "-sDEVICE=png16m -dTextAlphaBits=4 -dGraphicsAlphaBits=4") abs_src_subdir = os.path.join(self.WK_DIR, src_subdir) abs_dest_subdir = os.path.join(self.WK_DIR, dest_subdir) @@ -174,10 +174,9 @@ def convert_pod_figures(self, src_subdir, dest_subdir): # template for multi-page output). If input .ps/.pdf file has multiple # pages, this will generate 1 png per page, counting from 1. f_out = f_stem + '_MDTF_TEMP_%d.png' + cmd = f'gs {eps_convert_flags} -sOutputFile="{f_out}" {f}' try: - util.run_shell_command( - f'gs {eps_convert_flags} -sOutputFile="{f_out}" {f}' - ) + util.run_shell_command(cmd) except Exception as exc: self.obj.log.error("%s produced malformed plot: %s", self.obj.full_name, f[len(abs_src_subdir):]) diff --git a/tests/github_actions_test_macos_set1.jsonc b/tests/github_actions_test_macos_set1.jsonc index 6fb72fda8..e5ce236fa 100644 --- a/tests/github_actions_test_macos_set1.jsonc +++ b/tests/github_actions_test_macos_set1.jsonc @@ -42,7 +42,10 @@ // dependencies (path returned ls by running `conda info --base`.) If empty, // framework will attempt to determine location of system's conda installation. //"conda_root": "/Users/runner/miniconda3", - "conda_root": "/Users/runner/micromamba-bin", + "conda_root": "/Users/runner/micromamba", + + "micromamba_exe": "/Users/runner/micromamba-bin/micromamba", + // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left diff --git a/tests/github_actions_test_macos_set2.jsonc b/tests/github_actions_test_macos_set2.jsonc index 7c4af700e..c0c5af6f3 100644 --- a/tests/github_actions_test_macos_set2.jsonc +++ b/tests/github_actions_test_macos_set2.jsonc @@ -39,7 +39,9 @@ // dependencies (path returned ls by running `conda info --base`.) If empty, // framework will attempt to determine location of system's conda installation. //"conda_root": "/Users/runner/miniconda3", - "conda_root": "/Users/runner/micromamba-bin", + "conda_root": "/Users/runner/micromamba", + + "micromamba_exe": "/Users/runner/micromamba-bin/micromamba", // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left diff --git a/tests/github_actions_test_macos_set3.jsonc b/tests/github_actions_test_macos_set3.jsonc index c28e9b14d..e7cf318b3 100644 --- a/tests/github_actions_test_macos_set3.jsonc +++ b/tests/github_actions_test_macos_set3.jsonc @@ -45,7 +45,9 @@ // dependencies (path returned lsby running `conda info --base`.) If empty, // framework will attempt to determine location of system's conda installation. //"conda_root": "/Users/runner/miniconda3", - "conda_root": "/Users/runner/micromamba-bin", + "conda_root": "/Users/runner/micromamba", + + "micromamba_exe": "/Users/runner/micromamba-bin/micromamba", // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left diff --git a/tests/github_actions_test_ubuntu_set1.jsonc b/tests/github_actions_test_ubuntu_set1.jsonc index f0cde04d9..4e9282d7f 100644 --- a/tests/github_actions_test_ubuntu_set1.jsonc +++ b/tests/github_actions_test_ubuntu_set1.jsonc @@ -47,8 +47,9 @@ // dependencies (path returned lsby running `conda info --base`.) If empty, // framework will attempt to determine location of system's conda installation. //"conda_root": "/usr/share/miniconda3", - "conda_root": "/home/runner/micromamba-bin", + "conda_root": "/home/runner/micromamba", + "micromamba_exe": "/home/runner/micromamba-bin/micromamba", // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left // blank, the framework will look for its environments in the system default diff --git a/tests/github_actions_test_ubuntu_set2.jsonc b/tests/github_actions_test_ubuntu_set2.jsonc index e1d117967..0664e7dc1 100644 --- a/tests/github_actions_test_ubuntu_set2.jsonc +++ b/tests/github_actions_test_ubuntu_set2.jsonc @@ -37,7 +37,9 @@ // dependencies (path returned lsby running `conda info --base`.) If empty, // framework will attempt to determine location of system's conda installation. //"conda_root": "/usr/share/miniconda3", - "conda_root": "/home/runner/micromamba-bin", + "conda_root": "/home/runner/micromamba", + + "micromamba_exe": "/home/runner/micromamba-bin/micromamba", // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left diff --git a/tests/github_actions_test_ubuntu_set3.jsonc b/tests/github_actions_test_ubuntu_set3.jsonc index 135cc0763..e032fcee2 100644 --- a/tests/github_actions_test_ubuntu_set3.jsonc +++ b/tests/github_actions_test_ubuntu_set3.jsonc @@ -44,8 +44,9 @@ // dependencies (path returned lsby running `conda info --base`.) If empty, // framework will attempt to determine location of system's conda installation. //"conda_root": "/usr/share/miniconda3", - "conda_root": "/home/runner/micromamba-bin", + "conda_root": "/home/runner/micromamba", + "micromamba_exe": "/home/runner/micromamba-bin/micromamba", // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left // blank, the framework will look for its environments in the system default