From fceacc0ee1f4c5dac48b478035be3df7e982a894 Mon Sep 17 00:00:00 2001 From: Elliot Fontaine <92150839+elliotfontaine@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:48:15 +0200 Subject: [PATCH 1/3] implement conda envs validation --- cylc-src/bioreactor-workflow/envs/conda.cylc | 32 +++++++++-- .../bioreactor-workflow/meta/rose-meta.conf | 4 ++ cylc-src/bioreactor-workflow/rose-suite.conf | 56 +++++++++++-------- 3 files changed, 64 insertions(+), 28 deletions(-) diff --git a/cylc-src/bioreactor-workflow/envs/conda.cylc b/cylc-src/bioreactor-workflow/envs/conda.cylc index cebd326..d4ec4c2 100644 --- a/cylc-src/bioreactor-workflow/envs/conda.cylc +++ b/cylc-src/bioreactor-workflow/envs/conda.cylc @@ -8,16 +8,40 @@ 'CONDA_OPENMS': 'wf-pyopenms', } %} +[scheduling] + [[graph]] +{% if cfg__validate_conda_envs %} + R1/^ = validate_cfg => validate_conda => is_setup +{% endif %} + [runtime] -{% for env, conda_env_name in conda_envs.items() %} - [[{{env}}]] + +{% for FAMILY, conda_env_name in conda_envs.items() %} + [[{{FAMILY}}]] pre-script = """ set +eu conda activate {{ conda_env_name }} set -eu """ [[[meta]]] - title = "{{ env }}" + title = "{{ FAMILY }}" description = "Conda environment activation: {{ conda_env_name }}" categories = family task, conda -{% endfor %} \ No newline at end of file +{% endfor %} + + [[validate_conda]] + script = """ + >&2 echo "conda compare outputs errors to stdout. Check 'job.out' for errors." +{% for conda_env_name in conda_envs.values() %} + echo "Checking conda environment: {{ conda_env_name }}" + conda compare \ + ${CYLC_WORKFLOW_RUN_DIR}/envs/{{ conda_env_name }}.yml \ + -n {{ conda_env_name }} +{% endfor %} + """ + [[[meta]]] + title = "Validate Conda Environments" + description = """ + Check if the conda environments are up-to-date and all packages are present. + """ + categories = family task, conda diff --git a/cylc-src/bioreactor-workflow/meta/rose-meta.conf b/cylc-src/bioreactor-workflow/meta/rose-meta.conf index 1dfb499..4d4148a 100644 --- a/cylc-src/bioreactor-workflow/meta/rose-meta.conf +++ b/cylc-src/bioreactor-workflow/meta/rose-meta.conf @@ -31,6 +31,10 @@ compulsory=true range=0: type=real +[template variables=cfg__validate_conda_envs] +compulsory=true +type=python_boolean + [template variables=cfg__scans_window] compulsory=false pattern=^'(|(\d+)-(\d+))'$ diff --git a/cylc-src/bioreactor-workflow/rose-suite.conf b/cylc-src/bioreactor-workflow/rose-suite.conf index cf4cc69..137a577 100644 --- a/cylc-src/bioreactor-workflow/rose-suite.conf +++ b/cylc-src/bioreactor-workflow/rose-suite.conf @@ -8,7 +8,37 @@ # It is rendered directly from the docstrings here. -########## Scientific parameters ########## + +################################### Global workflow parameters #################################### + +# Should the workflow validate the conda environments for packages' presence +# and versions before running. +# +# :default: False +# :type: bool +cfg__validate_conda_envs = False + +# `internal | local`. +# +# * *internal*: will look for .raw files in :file:`${CYLC_WORKFLOW_RUN_DIR}/raws`. +# * *local*: will look for .raw files in :file:`{cfg__local_runs_dir}/${CYLC_WORKFLOW_RUN_NAME}`. +# +# :default: 'internal' +# :type: str +cfg__input_strategy='internal' + +# Used if :rose:conf:`cfg__input_strategy` is set to *local*. If there is a +# subdirectory in the one defined here that shares the same name as the +# :term:`workflow run`, the workflow will look for .raw files inside. +# +# :optional: yes +# :default: None +# :type: str (path) +cfg__local_runs_dir='' + + + +###################################### Scientific parameters ###################################### # Tuple for the number of scans to trim at the start and at the end of the # :term:`flowgram`. If set to [0, 0], no trimming will be done. @@ -42,29 +72,7 @@ cfg__ppm_tol=5 -########## Input Strategy ########## - -# `internal | local`. -# -# * *internal*: will look for .raw files in :file:`${CYLC_WORKFLOW_RUN_DIR}/raws`. -# * *local*: will look for .raw files in :file:`{cfg__local_runs_dir}/${CYLC_WORKFLOW_RUN_NAME}`. -# -# :default: 'internal' -# :type: str -cfg__input_strategy='internal' - -# Used if :rose:conf:`cfg__input_strategy` is set to *local*. If there is a -# subdirectory in the one defined here that shares the same name as the -# :term:`workflow run`, the workflow will look for .raw files inside. -# -# :optional: yes -# :default: None -# :type: str (path) -cfg__local_runs_dir='' - - - -########## [OPTIONAL] InfluxDB Setup ########## +#################################### [OPTIONAL] InfluxDB Setup #################################### # Should the workflow send data to an InfluxDB instance for visualization. # If set to *True*, the following configuration options must be set. From 3fd65c633b328491f2f4349ee254bff4e0e47e7e Mon Sep 17 00:00:00 2001 From: Elliot Fontaine <92150839+elliotfontaine@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:52:25 +0200 Subject: [PATCH 2/3] fix tutorial url --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d17008e..476215c 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ The raws files you provide should be numbered that way: The workflow will automatically detect the files and process them. Results (metabolites, concentrations, etc) are in `share/cycle/N/` of the run directory. -For a more detailed guide, see the [Tutorial](https://rtmet.readthedocs.io/en/latest/getting_started/tutorial.html) in the docs. +For a more detailed guide, see the [Tutorial](https://rtmet.readthedocs.io/en/latest/getting_started/basic_tutorial.html) in the docs. ## 🪲 Bugs and feature requests From 0c3bfb80f31ed1638882280d325cd75db0c92a8f Mon Sep 17 00:00:00 2001 From: Elliot Fontaine <92150839+elliotfontaine@users.noreply.github.com> Date: Mon, 22 Jul 2024 22:22:23 +0200 Subject: [PATCH 3/3] tidy up binneR env file --- cylc-src/bioreactor-workflow/envs/wf-binner.yml | 12 ++++-------- docs/source/getting_started/installation.rst | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cylc-src/bioreactor-workflow/envs/wf-binner.yml b/cylc-src/bioreactor-workflow/envs/wf-binner.yml index 034e24e..6e2b344 100644 --- a/cylc-src/bioreactor-workflow/envs/wf-binner.yml +++ b/cylc-src/bioreactor-workflow/envs/wf-binner.yml @@ -3,16 +3,12 @@ channels: - conda-forge - bioconda dependencies: - - r-base=4.3.1 - - r-essentials=4.3 - r-remotes=2.4 - - r-furrr=0.3.1 - - r-ggthemes=5.1.0 + - r-essentials=4.3 - r-optparse=1.7.4 - - r-tidyselect=1.2.1 - - r-rcpp=1.0.12 + - r-ggthemes=5.1.0 + - r-furrr=0.3.1 - bioconductor-mzr=2.36 - - bioconductor-rhdf5lib=1.24 # Inside R, install the binneR package from GitHub -# remotes::install_github('aberHRML/binneR') \ No newline at end of file +# conda activate wf-binner && Rscript -e "remotes::install_github('aberHRML/binneR', upgrade='never')" \ No newline at end of file diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index d5fdda5..409abfc 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -96,7 +96,7 @@ Bioinformatics tools are installed in separate Conda environments, for isolation purposes. For binneR, you'll need to install it from the R console:: $ for file in wf-*.yml; do conda env create -f $file; done - $ conda activate wf-binner && Rscript -e "remotes::install_github('aberHRML/binneR', dependencies=FALSE, upgrade_dependencies=FALSE)" + $ conda activate wf-binner && Rscript -e "remotes::install_github('aberHRML/binneR', upgrade='never')" $ conda deactivate .. _installation.influxdb: