From fe306539033a1c6af1d3087c959c509875213152 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 5 Jul 2023 09:44:04 +0200 Subject: [PATCH 1/5] remove docker volumes, make template --- files/galaxy/tpv/{tools.yml => tools.yml.j2} | 4 ---- 1 file changed, 4 deletions(-) rename files/galaxy/tpv/{tools.yml => tools.yml.j2} (92%) diff --git a/files/galaxy/tpv/tools.yml b/files/galaxy/tpv/tools.yml.j2 similarity index 92% rename from files/galaxy/tpv/tools.yml rename to files/galaxy/tpv/tools.yml.j2 index ae5a9af48..0faf37255 100644 --- a/files/galaxy/tpv/tools.yml +++ b/files/galaxy/tpv/tools.yml.j2 @@ -642,7 +642,6 @@ tools: params: submit_requirements: 'GalaxyGroup == "compute_mothur"' docker_run_extra_arguments: --pids-limit 10000 --ulimit fsize=1000000000 --env TERM=vt100 - docker_volumes: "$_CONDOR_SCRATCH_DIR:rw,$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/data/db/:ro,/data/dnb01/galaxy_db/:ro,/data/dnb02/galaxy_db/:ro,/data/dnb03/galaxy_db/:ro,/data/dnb05/galaxy_db/:ro,/data/dnb06/galaxy_db/:rw,/data/dnb07/galaxy_db/:rw,/data/dnb08/galaxy_db/:rw,/data/dnb-ds02/galaxy_db/:ro,/data/dp01/galaxy_db/:rw,/data/0/galaxy_db/:ro,/data/1/galaxy_db/:ro,/data/2/galaxy_db/:ro,/data/3/galaxy_db/:ro,/data/4/galaxy_db/:ro,/data/5/galaxy_import/galaxy_user_data/:ro,/data/6/galaxy_db/:ro,/data/7/galaxy_db/:ro,/usr/local/tools/:ro" docker_default_container_id: centos:8.3.2011 scheduling: require: @@ -656,7 +655,6 @@ tools: params: submit_requirements: 'GalaxyGroup == "compute_mothur"' docker_run_extra_arguments: --pids-limit 10000 --ulimit fsize=1000000000 --env TERM=vt100 - docker_volumes: "$_CONDOR_SCRATCH_DIR:rw,$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/data/db/:ro,/data/dnb01/galaxy_db/:ro,/data/dnb02/galaxy_db/:ro,/data/dnb03/galaxy_db/:ro,/data/dnb05/galaxy_db/:ro,/data/dnb06/galaxy_db/:rw,/data/dnb07/galaxy_db/:rw,/data/dnb08/galaxy_db/:rw,/data/dnb-ds02/galaxy_db/:ro,/data/dp01/galaxy_db/:rw,/data/0/galaxy_db/:ro,/data/1/galaxy_db/:ro,/data/2/galaxy_db/:ro,/data/3/galaxy_db/:ro,/data/4/galaxy_db/:ro,/data/5/galaxy_import/galaxy_user_data/:ro,/data/6/galaxy_db/:ro,/data/7/galaxy_db/:ro,/usr/local/tools/:ro" docker_default_container_id: centos:8.3.2011 scheduling: require: @@ -667,7 +665,6 @@ tools: '.*bioext_bam2msa.*': params: docker_run_extra_arguments: --pids-limit 10000 --ulimit fsize=1000000000 --env TERM=vt100 - docker_volumes: "$_CONDOR_SCRATCH_DIR:rw,$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/data/db/:ro,/data/dnb01/galaxy_db/:ro,/data/dnb02/galaxy_db/:ro,/data/dnb03/galaxy_db/:ro,/data/dnb05/galaxy_db/:ro,/data/dnb06/galaxy_db/:rw,/data/dnb07/galaxy_db/:rw,,/data/dnb08/galaxy_db/:rw,/data/dnb-ds02/galaxy_db/:ro,/data/dp01/galaxy_db/:rw,/data/0/galaxy_db/:ro,/data/1/galaxy_db/:ro,/data/2/galaxy_db/:ro,/data/3/galaxy_db/:ro,/data/4/galaxy_db/:ro,/data/5/galaxy_import/galaxy_user_data/:ro,/data/6/galaxy_db/:ro,/data/7/galaxy_db/:ro,/usr/local/tools/:ro" docker_default_container_id: centos:8.3.2011 scheduling: require: @@ -677,7 +674,6 @@ tools: 'last_*': params: docker_run_extra_arguments: --pids-limit 10000 --ulimit fsize=1000000000 --env TERM=vt100 - docker_volumes: "$_CONDOR_SCRATCH_DIR:rw,$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/data/db/:ro,/data/dnb01/galaxy_db/:ro,/data/dnb02/galaxy_db/:ro,/data/dnb03/galaxy_db/:ro,/data/dnb05/galaxy_db/:ro,/data/dnb06/galaxy_db/:rw,/data/dnb07/galaxy_db/:rw,/data/dp01/galaxy_db/:rw,/data/0/galaxy_db/:ro,/data/1/galaxy_db/:ro,/data/2/galaxy_db/:ro,/data/3/galaxy_db/:ro,/data/4/galaxy_db/:ro,/data/5/galaxy_import/galaxy_user_data/:ro,/data/6/galaxy_db/:ro,/data/7/galaxy_db/:ro,/usr/local/tools/:ro" docker_default_container_id: centos:8.3.2011 scheduling: require: From f7c0532305a6566cb743ec17f039bd323d65e7b2 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 5 Jul 2023 09:44:34 +0200 Subject: [PATCH 2/5] tools as template --- group_vars/sn06.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/sn06.yml b/group_vars/sn06.yml index 1f3efb308..aa060601e 100644 --- a/group_vars/sn06.yml +++ b/group_vars/sn06.yml @@ -542,8 +542,6 @@ galaxy_config_files: dest: "{{ galaxy_config_dir }}/echo_main_env.xml" - src: "{{ galaxy_config_file_src_dir }}/tpv/tool_defaults.yml" dest: "{{ tpv_mutable_dir }}/tool_defaults.yml" - - src: "{{ galaxy_config_file_src_dir }}/tpv/tools.yml" - dest: "{{ tpv_mutable_dir }}/tools.yml" - src: "{{ galaxy_config_file_src_dir }}/tpv/roles.yml" dest: "{{ tpv_mutable_dir }}/roles.yml" - src: "{{ galaxy_config_file_src_dir }}/tpv/interactive_tools.yml" @@ -594,3 +592,5 @@ galaxy_config_templates: dest: "{{ galaxy_config['galaxy']['object_store_config_file'] }}" - src: "{{ galaxy_config_file_src_dir }}/tpv/destinations.yml.j2" dest: "{{ tpv_mutable_dir }}/destinations.yml" + - src: "{{ galaxy_config_file_src_dir }}/tpv/tools.yml.j2" + dest: "{{ tpv_mutable_dir }}/tools.yml" From 7c0011f9171ad9a02f9d3c27e024682aa6375521 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 5 Jul 2023 09:45:16 +0200 Subject: [PATCH 3/5] python2 for srma --- files/galaxy/tpv/tools.yml.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/files/galaxy/tpv/tools.yml.j2 b/files/galaxy/tpv/tools.yml.j2 index 0faf37255..98cad6a07 100644 --- a/files/galaxy/tpv/tools.yml.j2 +++ b/files/galaxy/tpv/tools.yml.j2 @@ -886,3 +886,9 @@ tools: scheduling: require: - singularity + + ".*srma_wrapper.*": + scheduling: + require: + - singularity + singularity_default_container_id: "{{ cvmfs.singularity.path }}/all/python:2.7.16" From 0d38991b62a269986514243ae52d9703a4502072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Wed, 5 Jul 2023 11:20:28 +0200 Subject: [PATCH 4/5] Add workaround for files that are converted into templates to TPV dry-run workflow --- .github/workflows/tpv.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tpv.yml b/.github/workflows/tpv.yml index bb83c2705..8529957f7 100644 --- a/.github/workflows/tpv.yml +++ b/.github/workflows/tpv.yml @@ -303,6 +303,7 @@ jobs: import os import subprocess import sys + from pathlib import Path # import make_playbook from tpv.py spec = importlib.util.spec_from_file_location('tpv_ci', '.github/workflows/tpv.py') @@ -329,6 +330,12 @@ jobs: subprocess.run( ["git", "checkout", "${{ steps.commits-infrastructure-playbook.outputs.before }}"] ) + # workaround for files that are converted into templates + templates = tuple( + (src, dest) + for src, dest in templates + if Path(src).exists() + ) playbook = make_playbook('sn06.yml', templates=templates) with open(os.environ["GITHUB_OUTPUT"], "a") as file: file.write(f"old={playbook}\n") From 09b4d4179e4875799754e34daea3ebb21827ab5b Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:59:10 +0200 Subject: [PATCH 5/5] Update files/galaxy/tpv/tools.yml.j2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- files/galaxy/tpv/tools.yml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/files/galaxy/tpv/tools.yml.j2 b/files/galaxy/tpv/tools.yml.j2 index 98cad6a07..8db398db6 100644 --- a/files/galaxy/tpv/tools.yml.j2 +++ b/files/galaxy/tpv/tools.yml.j2 @@ -891,4 +891,5 @@ tools: scheduling: require: - singularity - singularity_default_container_id: "{{ cvmfs.singularity.path }}/all/python:2.7.16" + params: + singularity_default_container_id: "{{ cvmfs.singularity.path }}/all/python:2.7.16"