From e51ee3a1e91162e7fb399999ad27b5ce93093d81 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 10 Apr 2024 19:09:37 +0200 Subject: [PATCH 01/14] Implement autoupdate with alternative shed tools --- planemo/autoupdate.py | 84 +++++++++++++++++++++++------- planemo/commands/cmd_autoupdate.py | 8 +++ planemo/galaxy/config.py | 12 +++-- planemo/workflow_lint.py | 2 +- tests/test_autoupdate.py | 31 ++++++++++- 5 files changed, 111 insertions(+), 26 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index d79c5c726..f22a940fd 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -4,6 +4,8 @@ import itertools import re import xml.etree.ElementTree as ET +from functools import lru_cache +from string import Template from typing import ( Any, DefaultDict, @@ -19,7 +21,6 @@ import requests import yaml -from bioblend import toolshed from bioblend.toolshed import ToolShedInstance from galaxy.tool_util.deps import conda_util from galaxy.tool_util.version import parse_version @@ -28,6 +29,7 @@ from planemo.io import ( error, info, + warn, ) from planemo.workflow_lint import ( find_repos_from_tool_id, @@ -293,8 +295,31 @@ def get_newest_tool_id(tool_ids: List[str]) -> str: )[-1] +@lru_cache(maxsize=None) +def get_toolshed_url_for_tool_id(tool_id: str) -> Optional[str]: + components = tool_id.split("/repos") + if len(components) > 1: + tool_shed_fqdn = components[0] + if tool_shed_fqdn in MAIN_TOOLSHED_URL: + return MAIN_TOOLSHED_URL + else: + # guess if tool shed is served over https or http + https_tool_shed_url = f"https://{tool_shed_fqdn}" + r = requests.get(https_tool_shed_url) + if r.status_code == 200: + return https_tool_shed_url + else: + http_tool_shed_url = f"http://{tool_shed_fqdn}" + r = requests.get(http_tool_shed_url) + if r.status_code == 200: + return http_tool_shed_url + else: + warn(f"Could not connect to {tool_shed_fqdn}") + return None + + def outdated_tools( # noqa: C901 - ctx: "PlanemoCliContext", wf_dict: Dict[str, Any], ts: ToolShedInstance, tools_to_skip: List[str] + ctx: "PlanemoCliContext", wf_dict: Dict[str, Any], tools_to_skip: List[str] ) -> Dict[str, Dict[str, str]]: """ tools_to_skip should be a list of base tool ids. @@ -305,8 +330,12 @@ def base_tool_id(tool_id: str) -> str: def check_tool_step(tool_id: str) -> Dict[str, Dict[str, str]]: """ - Return a dict with current and newest tool version, in case they don't match + Return a dict with current and newest tool version, in case they don't match. """ + tool_shed_url = get_toolshed_url_for_tool_id(tool_id) + if not tool_shed_url: + return {} + ts = ToolShedInstance(tool_shed_url) warning_msg, repos = find_repos_from_tool_id(tool_id, ts) if warning_msg != "": ctx.log(warning_msg) @@ -328,20 +357,12 @@ def check_tool_step(tool_id: str) -> Dict[str, Dict[str, str]]: return {} def outdated_tools_rec(wf_dict: Dict[str, Any]) -> None: - steps = wf_dict["steps"].values() if isinstance(wf_dict["steps"], dict) else wf_dict["steps"] - for step in steps: - if step.get("type", "tool") == "tool" and not step.get("run", {}).get("class") == "GalaxyWorkflow": - tool_id = step["tool_id"] - base_id = base_tool_id(tool_id) - if base_id not in checked_tools: - outdated_tool_dict.update(check_tool_step(tool_id)) - checked_tools.append(base_id) - elif step.get("type") == "subworkflow": # GA SWF - outdated_tools_rec(step["subworkflow"]) - elif step.get("run", {}).get("class") == "GalaxyWorkflow": # gxformat2 SWF - outdated_tools_rec(step["run"]) - else: - continue + tool_ids = get_tool_ids_for_workflow(wf_dict) + for tool_id in tool_ids: + base_id = base_tool_id(tool_id) + if base_id not in checked_tools: + outdated_tool_dict.update(check_tool_step(tool_id)) + checked_tools.append(base_id) outdated_tool_dict: Dict[str, Dict[str, str]] = {} # Initialize the list of tools already checked with a copy of tools_to_skip @@ -350,6 +371,22 @@ def outdated_tools_rec(wf_dict: Dict[str, Any]) -> None: return outdated_tool_dict +def get_tool_ids_for_workflow(wf_dict: Dict[str, Any], tool_ids: Optional[List[str]] = None) -> List[str]: + tool_ids = [] if tool_ids is None else tool_ids + steps = wf_dict["steps"].values() if isinstance(wf_dict["steps"], dict) else wf_dict["steps"] + for step in steps: + if step.get("type", "tool") == "tool" and not step.get("run", {}).get("class") == "GalaxyWorkflow": + tool_id = step["tool_id"] + tool_ids.append(tool_id) + elif step.get("type") == "subworkflow": # GA SWF + get_tool_ids_for_workflow(step["subworkflow"], tool_ids=tool_ids) + elif step.get("run", {}).get("class") == "GalaxyWorkflow": # gxformat2 SWF + get_tool_ids_for_workflow(step["run"], tool_ids=tool_ids) + else: + continue + return list(dict.fromkeys(tool_ids)) + + def get_tools_to_update( ctx: "PlanemoCliContext", workflow: "Runnable", tools_to_skip: List[str] ) -> Dict[str, Dict[str, str]]: @@ -359,8 +396,17 @@ def get_tools_to_update( with open(workflow.path) as f: wf_dict = yaml.load(f, Loader=yaml.SafeLoader) - ts = toolshed.ToolShedInstance(url=MAIN_TOOLSHED_URL) - return outdated_tools(ctx, wf_dict, ts, tools_to_skip) + return outdated_tools(ctx, wf_dict, tools_to_skip) + + +def get_shed_tools_conf_string_for_tool_ids(tool_ids: List[str]) -> str: + tool_shed_urls = set(get_toolshed_url_for_tool_id(tool_id) for tool_id in tool_ids if tool_id) + TOOL_SHEDS_CONF_TEMPLATE = Template("""${tool_shed_lines}""") + tool_sheds: List[str] = [] + for tool_shed_url in tool_shed_urls: + if tool_shed_url: + tool_sheds.append(f'') + return TOOL_SHEDS_CONF_TEMPLATE.substitute(tool_shed_lines="".join(tool_sheds)) def autoupdate_wf(ctx: "PlanemoCliContext", config: "LocalGalaxyConfig", wf: "Runnable") -> Dict[str, Any]: diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 40c988810..464702b70 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -135,6 +135,14 @@ def cli(ctx, paths, **kwds): # noqa C901 kwds["install_repository_dependencies"] = False kwds["shed_install"] = True + tool_ids = [] + for workflow in modified_workflows: + with open(workflow.path) as fh: + wf_dict = yaml.safe_load(fh) + tool_ids.extend(autoupdate.get_tool_ids_for_workflow(wf_dict=wf_dict)) + tool_ids = list(dict.fromkeys(tool_ids)) + kwds["tool_sheds_config_content"] = autoupdate.get_shed_tools_conf_string_for_tool_ids(tool_ids) + with engine_context(ctx, **kwds) as galaxy_engine: with galaxy_engine.ensure_runnables_served(modified_workflows) as config: for workflow in modified_workflows: diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index caf0c768c..ec7c13409 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -1202,11 +1202,13 @@ def _search_tool_path_for(path, target, extra_paths=None): def _configure_sheds_config_file(ctx, config_directory, **kwds): - if "shed_target" not in kwds: - kwds = kwds.copy() - kwds["shed_target"] = "toolshed" - shed_target_url = tool_shed_url(ctx, **kwds) - contents = _sub(TOOL_SHEDS_CONF, {"shed_target_url": shed_target_url}) + contents = kwds.get("tool_sheds_config_content") + if not contents: + if "shed_target" not in kwds: + kwds = kwds.copy() + kwds["shed_target"] = "toolshed" + shed_target_url = tool_shed_url(ctx, **kwds) + contents = _sub(TOOL_SHEDS_CONF, {"shed_target_url": shed_target_url}) tool_sheds_conf = os.path.join(config_directory, "tool_sheds_conf.xml") write_file(tool_sheds_conf, contents) return tool_sheds_conf diff --git a/planemo/workflow_lint.py b/planemo/workflow_lint.py index ce1439c70..344f30d64 100644 --- a/planemo/workflow_lint.py +++ b/planemo/workflow_lint.py @@ -479,7 +479,7 @@ def find_repos_from_tool_id(tool_id: str, ts: ToolShedInstance) -> Tuple[str, Di """ Return a string which indicates what failed and dict with all revisions for a given tool id """ - if not tool_id.startswith(MAIN_TOOLSHED_URL[8:]): + if "/repos" not in tool_id: return ("", {}) # assume a built in tool try: repos = ts.repositories._get(params={"tool_ids": tool_id}) diff --git a/tests/test_autoupdate.py b/tests/test_autoupdate.py index b49b51ac2..71ecc128a 100644 --- a/tests/test_autoupdate.py +++ b/tests/test_autoupdate.py @@ -1,4 +1,13 @@ -from planemo.autoupdate import get_newest_tool_id +import json +import os +from galaxy.util import parse_xml_string +from planemo.autoupdate import ( + get_newest_tool_id, + get_shed_tools_conf_string_for_tool_ids, + get_tool_ids_for_workflow, + get_toolshed_url_for_tool_id, +) +from .test_utils import TEST_DATA_DIR def test_get_newest_tool_id(): @@ -7,3 +16,23 @@ def test_get_newest_tool_id(): "toolshed.g2.bx.psu.edu/repos/iuc/rgrnastar/rna_star/2.7.8a", ] assert get_newest_tool_id(tool_ids) == tool_ids[0] + + +def test_get_toolshed_url_for_tool_id(): + assert get_toolshed_url_for_tool_id("github.com/repos/iuc/tool/1") == "https://github.com" + + +def test_get_shed_tools_conf_string_for_workflow(): + test_artifact = os.path.join(TEST_DATA_DIR, "wf_same_name_outputs.ga") + with open(test_artifact) as wf: + wf_dict = json.load(wf) + wf_dict["steps"]["3"]["tool_id"] = wf_dict["steps"]["3"]["tool_id"].replace("toolshed", "testtoolshed") + tool_ids = get_tool_ids_for_workflow(wf_dict=wf_dict) + xml_string = get_shed_tools_conf_string_for_tool_ids(tool_ids) + element = parse_xml_string(xml_string) + tool_shed_entries = element.findall("tool_shed") + assert len(tool_shed_entries) == 2 + assert tool_shed_entries[0].attrib["url"] == "https://toolshed.g2.bx.psu.edu" + assert tool_shed_entries[0].attrib["name"] == "toolshed.g2.bx.psu.edu" + assert tool_shed_entries[1].attrib["url"] == "https://testtoolshed.g2.bx.psu.edu" + assert tool_shed_entries[1].attrib["name"] == "testtoolshed.g2.bx.psu.edu" From cd735fcf8cb4bc484dec737e76b95a52f43be323 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 10 Apr 2024 19:33:00 +0200 Subject: [PATCH 02/14] Fix for new toolshed ? --- planemo/autoupdate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index f22a940fd..007385473 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -343,7 +343,9 @@ def check_tool_step(tool_id: str) -> Dict[str, Dict[str, str]]: return repos base_id = base_tool_id(tool_id) matching_tool_ids = [] - for repo in repos.values(): + # newer toolshed returns list + repos = repos.values() if isinstance(repos, dict) else repos + for repo in repos: if isinstance(repo, dict): for tool in repo.get("tools") or []: if tool["guid"].startswith(base_id + "/"): From d3e4b716c64b55d78bd8428c47fdb8c338350c3a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 10 Apr 2024 20:26:45 +0200 Subject: [PATCH 03/14] Make find_repos_from_tool_id fit for new toolshed --- planemo/workflow_lint.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/planemo/workflow_lint.py b/planemo/workflow_lint.py index 344f30d64..93309ca8d 100644 --- a/planemo/workflow_lint.py +++ b/planemo/workflow_lint.py @@ -481,10 +481,13 @@ def find_repos_from_tool_id(tool_id: str, ts: ToolShedInstance) -> Tuple[str, Di """ if "/repos" not in tool_id: return ("", {}) # assume a built in tool + *_, owner, name, _tool_id, _version = tool_id.split("/") + try: - repos = ts.repositories._get(params={"tool_ids": tool_id}) - except Exception: - return (f"The ToolShed returned an error when searching for the most recent version of {tool_id}", {}) + repo = ts.repositories.get_repositories(name, owner)[0] + repos = ts.repositories._get(url=f'{ts.repositories._make_url()}/{repo["id"]}/metadata') + except Exception as e: + return (f"The ToolShed returned an error when searching for the most recent version of {tool_id}: {e}", {}) if len(repos) == 0: return (f"The tool {tool_id} is not in the toolshed (may have been tagged as invalid).", {}) else: From db081c77e1785d633860fc0c719e5fcb2573756d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 00:19:47 +0200 Subject: [PATCH 04/14] Fix tool shed selection --- planemo/autoupdate.py | 35 +++++++++++------------------------ planemo/galaxy/workflows.py | 30 ++++++++++++++++++++++++++++++ planemo/workflow_lint.py | 3 +-- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 007385473..280811ffa 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -4,7 +4,6 @@ import itertools import re import xml.etree.ElementTree as ET -from functools import lru_cache from string import Template from typing import ( Any, @@ -26,15 +25,15 @@ from galaxy.tool_util.version import parse_version import planemo.conda +from planemo.galaxy.workflows import ( + guess_tool_shed_url, + MAIN_TOOLSHED_URL, +) from planemo.io import ( error, info, - warn, -) -from planemo.workflow_lint import ( - find_repos_from_tool_id, - MAIN_TOOLSHED_URL, ) +from planemo.workflow_lint import find_repos_from_tool_id if TYPE_CHECKING: from planemo.cli import PlanemoCliContext @@ -295,26 +294,11 @@ def get_newest_tool_id(tool_ids: List[str]) -> str: )[-1] -@lru_cache(maxsize=None) def get_toolshed_url_for_tool_id(tool_id: str) -> Optional[str]: components = tool_id.split("/repos") if len(components) > 1: tool_shed_fqdn = components[0] - if tool_shed_fqdn in MAIN_TOOLSHED_URL: - return MAIN_TOOLSHED_URL - else: - # guess if tool shed is served over https or http - https_tool_shed_url = f"https://{tool_shed_fqdn}" - r = requests.get(https_tool_shed_url) - if r.status_code == 200: - return https_tool_shed_url - else: - http_tool_shed_url = f"http://{tool_shed_fqdn}" - r = requests.get(http_tool_shed_url) - if r.status_code == 200: - return http_tool_shed_url - else: - warn(f"Could not connect to {tool_shed_fqdn}") + return guess_tool_shed_url(tool_shed_fqdn=tool_shed_fqdn) return None @@ -403,11 +387,14 @@ def get_tools_to_update( def get_shed_tools_conf_string_for_tool_ids(tool_ids: List[str]) -> str: tool_shed_urls = set(get_toolshed_url_for_tool_id(tool_id) for tool_id in tool_ids if tool_id) + tool_shed_urls = [_ for _ in tool_shed_urls if _ is not None] TOOL_SHEDS_CONF_TEMPLATE = Template("""${tool_shed_lines}""") tool_sheds: List[str] = [] - for tool_shed_url in tool_shed_urls: + # sort tool_shed_urls from shortest to longest, as https://github.com/galaxyproject/galaxy/blob/c7cb47a1b18ccd5b39075a705bbd2f34572755fe/lib/galaxy/util/tool_shed/tool_shed_registry.py#L106-L118 + # has a bug where a toolshed that is an exact substring of another registered toolshed would wrongly be selected. + for tool_shed_url in sorted(tool_shed_urls, key=lambda url: len(url)): if tool_shed_url: - tool_sheds.append(f'') + tool_sheds.append(f'') return TOOL_SHEDS_CONF_TEMPLATE.substitute(tool_shed_lines="".join(tool_sheds)) diff --git a/planemo/galaxy/workflows.py b/planemo/galaxy/workflows.py index 50bd8da54..9ad6e9abb 100644 --- a/planemo/galaxy/workflows.py +++ b/planemo/galaxy/workflows.py @@ -3,14 +3,17 @@ import json import os from collections import namedtuple +from functools import lru_cache from typing import ( Any, Callable, Dict, List, + Optional, ) from urllib.parse import urlparse +import requests import yaml from ephemeris import ( generate_tool_list_from_ga_workflow_files, @@ -35,6 +38,27 @@ FAILED_REPOSITORIES_MESSAGE = "Failed to install one or more repositories." GALAXY_WORKFLOWS_PREFIX = "gxid://workflows/" GALAXY_WORKFLOW_INSTANCE_PREFIX = "gxid://workflow-instance/" +MAIN_TOOLSHED_URL = "https://toolshed.g2.bx.psu.edu" + + +@lru_cache(maxsize=None) +def guess_tool_shed_url(tool_shed_fqdn: str) -> Optional[str]: + if tool_shed_fqdn in MAIN_TOOLSHED_URL: + return MAIN_TOOLSHED_URL + else: + # guess if tool shed is served over https or http + https_tool_shed_url = f"https://{tool_shed_fqdn}" + r = requests.get(https_tool_shed_url) + if r.status_code == 200: + return https_tool_shed_url + else: + http_tool_shed_url = f"http://{tool_shed_fqdn}" + r = requests.get(http_tool_shed_url) + if r.status_code == 200: + return http_tool_shed_url + else: + warn(f"Could not connect to {tool_shed_fqdn}") + return None def load_shed_repos(runnable): @@ -62,6 +86,12 @@ def load_shed_repos(runnable): if repository: repository["tool_panel_section_label"] = "Tools from workflows" tools.append(repository) + for repo in tools: + tool_shed = repo.get("tool_shed") + if tool_shed: + tool_shed_url = guess_tool_shed_url(tool_shed) + if tool_shed_url: + repo["tool_shed_url"] = tool_shed_url return tools diff --git a/planemo/workflow_lint.py b/planemo/workflow_lint.py index 93309ca8d..e79ddaef2 100644 --- a/planemo/workflow_lint.py +++ b/planemo/workflow_lint.py @@ -35,6 +35,7 @@ ) from planemo.galaxy.workflows import ( input_labels, + MAIN_TOOLSHED_URL, output_labels, required_input_labels, ) @@ -51,8 +52,6 @@ POTENTIAL_WORKFLOW_FILES = re.compile(r"^.*(\.yml|\.yaml|\.ga)$") DOCKSTORE_REGISTRY_CONF_VERSION = "1.2" -MAIN_TOOLSHED_URL = "https://toolshed.g2.bx.psu.edu" - class WorkflowLintContext(LintContext): # Setup training topic for linting - probably should pass this through From 3746c6877bd88ce1c4707bd2471fd80284f0cd9a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 00:21:53 +0200 Subject: [PATCH 05/14] Add test for workflow update from mixed tool sheds --- .../wf_autoupdate_test_multiple_repos.ga | 93 +++++++++++++++++++ tests/test_autoupdate.py | 2 + tests/test_cmd_autoupdate.py | 13 +++ 3 files changed, 108 insertions(+) create mode 100644 tests/data/wf_repos/autoupdate_tests/wf_autoupdate_test_multiple_repos.ga diff --git a/tests/data/wf_repos/autoupdate_tests/wf_autoupdate_test_multiple_repos.ga b/tests/data/wf_repos/autoupdate_tests/wf_autoupdate_test_multiple_repos.ga new file mode 100644 index 000000000..a6cc59cc5 --- /dev/null +++ b/tests/data/wf_repos/autoupdate_tests/wf_autoupdate_test_multiple_repos.ga @@ -0,0 +1,93 @@ +{ + "steps": { + "0": { + "collection_type": "paired", + "type": "data_collection_input", + "label": "pe-fastq", + "id": 0, + "position": { + "left": 0, + "top": 0 + }, + "annotation": "", + "input_connections": {}, + "inputs": [ + { + "name": "pe-fastq", + "description": "" + } + ], + "tool_state": "{\"name\": \"pe-fastq\", \"collection_type\": \"paired\"}" + }, + "1": { + "label": "fastqc 1", + "tool_id": "toolshed.g2.bx.psu.edu/repos/bgruening/text_processing/tp_find_and_replace/9.3+galaxy0", + "tool_version": "9.3+galaxy0", + "tool_shed_repository": { + "name": "text_processing", + "owner": "bgruening", + "tool_shed": "toolshed.g2.bx.psu.edu", + "changeset_revision": "12615d397df7" + }, + "id": 1, + "position": { + "left": 10, + "top": 10 + }, + "type": "tool", + "name": "toolshed.g2.bx.psu.edu/repos/bgruening/text_processing/tp_find_and_replace/9.3+galaxy0", + "post_job_actions": {}, + "annotation": "", + "input_connections": { + "input_file": [ + { + "id": 0, + "output_name": "output" + } + ] + }, + "tool_state": "{\"__page__\": 0}", + "workflow_outputs": [ + { + "output_name": "text_file", + "label": "FastQC txt", + "uuid": null + } + ] + }, + "2": { + "label": "fastqc 2", + "tool_id": "testtoolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.69", + "tool_version": "0.69", + "tool_shed_repository": { + "name": "fastqc", + "owner": "devteam", + "tool_shed": "testtoolshed.g2.bx.psu.edu", + "changeset_revision": "7b0207bd2183" + }, + "id": 2, + "position": { + "left": 20, + "top": 20 + }, + "type": "tool", + "name": "testtoolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.69", + "post_job_actions": {}, + "annotation": "", + "input_connections": { + "input_file": [ + { + "id": 0, + "output_name": "output" + } + ] + }, + "tool_state": "{\"__page__\": 0}" + } + }, + "a_galaxy_workflow": "true", + "format-version": "0.1", + "name": "Workflow", + "uuid": "869b6bb3-23a8-4322-a325-cc55abdf7540", + "annotation": "Run fastqc" +} diff --git a/tests/test_autoupdate.py b/tests/test_autoupdate.py index 71ecc128a..005244ed6 100644 --- a/tests/test_autoupdate.py +++ b/tests/test_autoupdate.py @@ -1,6 +1,8 @@ import json import os + from galaxy.util import parse_xml_string + from planemo.autoupdate import ( get_newest_tool_id, get_shed_tools_conf_string_for_tool_ids, diff --git a/tests/test_cmd_autoupdate.py b/tests/test_cmd_autoupdate.py index da4a90dfb..0229d9784 100644 --- a/tests/test_cmd_autoupdate.py +++ b/tests/test_cmd_autoupdate.py @@ -115,6 +115,19 @@ def test_autoupdate_workflow(self): assert wf["steps"][1]["run"]["steps"][0]["tool_version"] == "3.7+galaxy0" assert wf["release"] == "0.1.1" + def test_autoupdate_workflow_from_multiple_tool_sheds(self): + with self._isolate_with_test_data("wf_repos/autoupdate_tests") as f: + wf_file = os.path.realpath(os.path.join(f, "wf_autoupdate_test_multiple_repos.ga")) + autoupdate_command = ["autoupdate", wf_file] + result = self._runner.invoke(self._cli.planemo, autoupdate_command) + assert f"Auto-updating workflow {wf_file}" in result.output + with open(wf_file) as g: + wf = json.load(g) + # Assert toolshed tool is updated + assert wf["steps"]["1"]["tool_version"] != "9.3+galaxy0" + # Assert testtoolshed tool is updated + assert wf["steps"]["2"]["tool_version"] != "0.69" + def test_autoupdate_workflow_unexisting_version(self): """Test autoupdate command for a workflow where the version of the tool is not in the toolshed.""" with self._isolate_with_test_data("wf_repos/autoupdate_tests") as f: From 8595028f352f65b7c11855cdbde6f4f097c2c0c3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 00:37:11 +0200 Subject: [PATCH 06/14] Type fixes --- planemo/autoupdate.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 280811ffa..d328268dc 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -328,8 +328,7 @@ def check_tool_step(tool_id: str) -> Dict[str, Dict[str, str]]: base_id = base_tool_id(tool_id) matching_tool_ids = [] # newer toolshed returns list - repos = repos.values() if isinstance(repos, dict) else repos - for repo in repos: + for repo in repos.values(): if isinstance(repo, dict): for tool in repo.get("tools") or []: if tool["guid"].startswith(base_id + "/"): @@ -387,14 +386,13 @@ def get_tools_to_update( def get_shed_tools_conf_string_for_tool_ids(tool_ids: List[str]) -> str: tool_shed_urls = set(get_toolshed_url_for_tool_id(tool_id) for tool_id in tool_ids if tool_id) - tool_shed_urls = [_ for _ in tool_shed_urls if _ is not None] + cleaned_tool_shed_urls = set(_ for _ in tool_shed_urls if _ is not None) TOOL_SHEDS_CONF_TEMPLATE = Template("""${tool_shed_lines}""") tool_sheds: List[str] = [] # sort tool_shed_urls from shortest to longest, as https://github.com/galaxyproject/galaxy/blob/c7cb47a1b18ccd5b39075a705bbd2f34572755fe/lib/galaxy/util/tool_shed/tool_shed_registry.py#L106-L118 # has a bug where a toolshed that is an exact substring of another registered toolshed would wrongly be selected. - for tool_shed_url in sorted(tool_shed_urls, key=lambda url: len(url)): - if tool_shed_url: - tool_sheds.append(f'') + for tool_shed_url in sorted(cleaned_tool_shed_urls, key=lambda url: len(url)): + tool_sheds.append(f'') return TOOL_SHEDS_CONF_TEMPLATE.substitute(tool_shed_lines="".join(tool_sheds)) From 61c315aa6e26ab58f7d184e90daeb3ff51c13cf9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 09:31:58 +0200 Subject: [PATCH 07/14] Drop outdated comment --- planemo/autoupdate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index d328268dc..2d139ab98 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -327,7 +327,6 @@ def check_tool_step(tool_id: str) -> Dict[str, Dict[str, str]]: return repos base_id = base_tool_id(tool_id) matching_tool_ids = [] - # newer toolshed returns list for repo in repos.values(): if isinstance(repo, dict): for tool in repo.get("tools") or []: From 968a6b88d5066bd7cae185faac18d283f4edfc8e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 09:44:26 +0200 Subject: [PATCH 08/14] Fix tool lint check for correct tool version --- planemo/workflow_lint.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/planemo/workflow_lint.py b/planemo/workflow_lint.py index e79ddaef2..277e08bd9 100644 --- a/planemo/workflow_lint.py +++ b/planemo/workflow_lint.py @@ -493,6 +493,18 @@ def find_repos_from_tool_id(tool_id: str, ts: ToolShedInstance) -> Tuple[str, Di return ("", repos) +def assert_valid_tool_id_in_tool_shed(tool_id, ts: ToolShedInstance) -> Optional[str]: + warning_msg, repos = find_repos_from_tool_id(tool_id, ts) + if warning_msg: + return warning_msg + for repo in repos.values(): + tools = repo.get("tools", []) + for tool in tools: + if tool_id == tool.get("guid"): + return None + return f"The tool {tool_id} is not in the toolshed (may have been tagged as invalid)." + + def _lint_tool_ids(path: str, lint_context: WorkflowLintContext) -> None: def _lint_tool_ids_steps(lint_context: WorkflowLintContext, wf_dict: Dict, ts: ToolShedInstance) -> bool: """Returns whether a single tool_id was invalid""" @@ -500,8 +512,8 @@ def _lint_tool_ids_steps(lint_context: WorkflowLintContext, wf_dict: Dict, ts: T steps = wf_dict.get("steps", {}) for step in steps.values(): if step.get("type", "tool") == "tool" and not step.get("run", {}).get("class") == "GalaxyWorkflow": - warning_msg, _ = find_repos_from_tool_id(step["tool_id"], ts) - if warning_msg != "": + warning_msg = assert_valid_tool_id_in_tool_shed(step["tool_id"], ts) + if warning_msg: lint_context.error(warning_msg) failed = True elif step.get("type") == "subworkflow": # GA SWF @@ -521,5 +533,5 @@ def _lint_tool_ids_steps(lint_context: WorkflowLintContext, wf_dict: Dict, ts: T ts = toolshed.ToolShedInstance(url=MAIN_TOOLSHED_URL) failed = _lint_tool_ids_steps(lint_context, workflow_dict, ts) if not failed: - lint_context.valid("All tools_id appear to be valid.") + lint_context.valid("All tool ids appear to be valid.") return None From 62cec5ddb756f3745c72681420db15e144dba058 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 10:14:37 +0200 Subject: [PATCH 09/14] Use tox-managed virtualenv in make lint-docs --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index a636a5a56..f88d01d6f 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,7 @@ setenv = 2109: PLANEMO_TEST_GALAXY_BRANCH=release_21.09 2101: PLANEMO_TEST_GALAXY_BRANCH=release_21.01 2205: PLANEMO_TEST_GALAXY_BRANCH=release_22.05 + lint_docs: VENV={env:VIRTUAL_ENV} skip_install = doc_test,lint,lint_docs,lint_docstrings,mypy,gxwf_test_test: True allowlist_externals = From 97003c8f238530a6d7932b48c31887498f06b58d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 10:17:10 +0200 Subject: [PATCH 10/14] Don't fail linting on local tools --- planemo/workflow_lint.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/planemo/workflow_lint.py b/planemo/workflow_lint.py index 277e08bd9..fdc2bbc80 100644 --- a/planemo/workflow_lint.py +++ b/planemo/workflow_lint.py @@ -493,7 +493,9 @@ def find_repos_from_tool_id(tool_id: str, ts: ToolShedInstance) -> Tuple[str, Di return ("", repos) -def assert_valid_tool_id_in_tool_shed(tool_id, ts: ToolShedInstance) -> Optional[str]: +def assert_valid_tool_id_in_tool_shed(tool_id: str, ts: ToolShedInstance) -> Optional[str]: + if "/repos" not in tool_id: + return None warning_msg, repos = find_repos_from_tool_id(tool_id, ts) if warning_msg: return warning_msg From 1e81b3ec4087faf19ad9e92b93d9c716d3b73392 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Apr 2024 11:04:11 +0200 Subject: [PATCH 11/14] Ignore toil cwltool warning in lint_sphinx_output.py --- scripts/lint_sphinx_output.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/lint_sphinx_output.py b/scripts/lint_sphinx_output.py index 701479178..fe8ea002f 100644 --- a/scripts/lint_sphinx_output.py +++ b/scripts/lint_sphinx_output.py @@ -8,7 +8,7 @@ def warning_line(line): - if "WARNING" not in line: + if "WARNING" not in line or "Toil" in line: return False if "docs/tests" in line: # Doesn't actually show up in docs so don't lint. return False @@ -23,9 +23,7 @@ def main(argv=None): argv = sys.argv sphinx_output = sys.stdin.read() warning_lines = [_ for _ in sphinx_output.splitlines() if warning_line(_)] - for line in warning_lines: - print(line) - sys.exit(1 if warning_lines else 0) + sys.exit("\n".join(warning_lines) if warning_lines else 0) if __name__ == "__main__": From 1889bb22c6a98a792fe8c72c674c12a7a4066674 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 3 May 2024 13:10:11 +0200 Subject: [PATCH 12/14] Generalize finding tool sheds from runnable Should now work for workflow_edit, test, run etc if targeting a local or docker instance. --- planemo/autoupdate.py | 40 +------------------- planemo/commands/cmd_autoupdate.py | 8 ---- planemo/galaxy/config.py | 61 +++++++++++++++++++++++++----- planemo/galaxy/workflows.py | 37 ++++++++++++------ tests/test_autoupdate.py | 6 +-- 5 files changed, 81 insertions(+), 71 deletions(-) diff --git a/planemo/autoupdate.py b/planemo/autoupdate.py index 2d139ab98..1a9324d6b 100644 --- a/planemo/autoupdate.py +++ b/planemo/autoupdate.py @@ -4,7 +4,6 @@ import itertools import re import xml.etree.ElementTree as ET -from string import Template from typing import ( Any, DefaultDict, @@ -26,7 +25,8 @@ import planemo.conda from planemo.galaxy.workflows import ( - guess_tool_shed_url, + get_tool_ids_for_workflow, + get_toolshed_url_for_tool_id, MAIN_TOOLSHED_URL, ) from planemo.io import ( @@ -294,14 +294,6 @@ def get_newest_tool_id(tool_ids: List[str]) -> str: )[-1] -def get_toolshed_url_for_tool_id(tool_id: str) -> Optional[str]: - components = tool_id.split("/repos") - if len(components) > 1: - tool_shed_fqdn = components[0] - return guess_tool_shed_url(tool_shed_fqdn=tool_shed_fqdn) - return None - - def outdated_tools( # noqa: C901 ctx: "PlanemoCliContext", wf_dict: Dict[str, Any], tools_to_skip: List[str] ) -> Dict[str, Dict[str, str]]: @@ -355,22 +347,6 @@ def outdated_tools_rec(wf_dict: Dict[str, Any]) -> None: return outdated_tool_dict -def get_tool_ids_for_workflow(wf_dict: Dict[str, Any], tool_ids: Optional[List[str]] = None) -> List[str]: - tool_ids = [] if tool_ids is None else tool_ids - steps = wf_dict["steps"].values() if isinstance(wf_dict["steps"], dict) else wf_dict["steps"] - for step in steps: - if step.get("type", "tool") == "tool" and not step.get("run", {}).get("class") == "GalaxyWorkflow": - tool_id = step["tool_id"] - tool_ids.append(tool_id) - elif step.get("type") == "subworkflow": # GA SWF - get_tool_ids_for_workflow(step["subworkflow"], tool_ids=tool_ids) - elif step.get("run", {}).get("class") == "GalaxyWorkflow": # gxformat2 SWF - get_tool_ids_for_workflow(step["run"], tool_ids=tool_ids) - else: - continue - return list(dict.fromkeys(tool_ids)) - - def get_tools_to_update( ctx: "PlanemoCliContext", workflow: "Runnable", tools_to_skip: List[str] ) -> Dict[str, Dict[str, str]]: @@ -383,18 +359,6 @@ def get_tools_to_update( return outdated_tools(ctx, wf_dict, tools_to_skip) -def get_shed_tools_conf_string_for_tool_ids(tool_ids: List[str]) -> str: - tool_shed_urls = set(get_toolshed_url_for_tool_id(tool_id) for tool_id in tool_ids if tool_id) - cleaned_tool_shed_urls = set(_ for _ in tool_shed_urls if _ is not None) - TOOL_SHEDS_CONF_TEMPLATE = Template("""${tool_shed_lines}""") - tool_sheds: List[str] = [] - # sort tool_shed_urls from shortest to longest, as https://github.com/galaxyproject/galaxy/blob/c7cb47a1b18ccd5b39075a705bbd2f34572755fe/lib/galaxy/util/tool_shed/tool_shed_registry.py#L106-L118 - # has a bug where a toolshed that is an exact substring of another registered toolshed would wrongly be selected. - for tool_shed_url in sorted(cleaned_tool_shed_urls, key=lambda url: len(url)): - tool_sheds.append(f'') - return TOOL_SHEDS_CONF_TEMPLATE.substitute(tool_shed_lines="".join(tool_sheds)) - - def autoupdate_wf(ctx: "PlanemoCliContext", config: "LocalGalaxyConfig", wf: "Runnable") -> Dict[str, Any]: workflow_id = config.workflow_id_for_runnable(wf) _update_wf(config, workflow_id) diff --git a/planemo/commands/cmd_autoupdate.py b/planemo/commands/cmd_autoupdate.py index 464702b70..40c988810 100644 --- a/planemo/commands/cmd_autoupdate.py +++ b/planemo/commands/cmd_autoupdate.py @@ -135,14 +135,6 @@ def cli(ctx, paths, **kwds): # noqa C901 kwds["install_repository_dependencies"] = False kwds["shed_install"] = True - tool_ids = [] - for workflow in modified_workflows: - with open(workflow.path) as fh: - wf_dict = yaml.safe_load(fh) - tool_ids.extend(autoupdate.get_tool_ids_for_workflow(wf_dict=wf_dict)) - tool_ids = list(dict.fromkeys(tool_ids)) - kwds["tool_sheds_config_content"] = autoupdate.get_shed_tools_conf_string_for_tool_ids(tool_ids) - with engine_context(ctx, **kwds) as galaxy_engine: with galaxy_engine.ensure_runnables_served(modified_workflows) as config: for workflow in modified_workflows: diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index ec7c13409..98f518a70 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -35,7 +35,10 @@ from planemo.config import OptionSource from planemo.deps import ensure_dependency_resolvers_conf_configured from planemo.docker import docker_host_args -from planemo.galaxy.workflows import remote_runnable_to_workflow_id +from planemo.galaxy.workflows import ( + get_toolshed_url_for_tool_id, + remote_runnable_to_workflow_id, +) from planemo.io import ( communicate, kill_pid_file, @@ -48,6 +51,7 @@ write_file, ) from planemo.mulled import build_involucro_context +from planemo.runnable import RunnableType from planemo.shed import tool_shed_url from .api import ( DEFAULT_ADMIN_API_KEY, @@ -258,6 +262,10 @@ def config_join(*args): shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools") _ensure_directory(shed_tool_path) + # Find tool sheds to add to config + tool_sheds_config_content = get_tool_sheds_conf_for_runnables(runnables) + if tool_sheds_config_content: + kwds["tool_sheds_config_content"] = tool_sheds_config_content sheds_config_path = _configure_sheds_config_file(ctx, config_directory, **kwds) port = _get_port(kwds) properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests) @@ -389,6 +397,10 @@ def config_join(*args): shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools") _ensure_directory(shed_tool_path) + # Find tool sheds to add to config + tool_sheds_config_content = get_tool_sheds_conf_for_runnables(runnables) + if tool_sheds_config_content: + kwds["tool_sheds_config_content"] = tool_sheds_config_content sheds_config_path = _configure_sheds_config_file(ctx, config_directory, **kwds) database_location = config_join("galaxy.sqlite") @@ -568,19 +580,27 @@ def _all_tool_paths( all_tool_paths = {r.path for r in runnables if r.has_tools and not r.data_manager_conf_path} extra_tools = _expand_paths(galaxy_root, extra_tools=extra_tools) all_tool_paths.update(extra_tools) - for runnable in runnables: - if runnable.type.name == "galaxy_workflow": - tool_ids = find_tool_ids(runnable.path) - for tool_id in tool_ids: - tool_paths = DISTRO_TOOLS_ID_TO_PATH.get(tool_id) - if tool_paths: - if isinstance(tool_paths, str): - tool_paths = [tool_paths] - all_tool_paths.update(tool_paths) + for tool_id in get_tool_ids_for_runnables(runnables): + tool_paths = DISTRO_TOOLS_ID_TO_PATH.get(tool_id) + if tool_paths: + if isinstance(tool_paths, str): + tool_paths = [tool_paths] + all_tool_paths.update(tool_paths) return all_tool_paths +def get_workflow_runnables(runnables: List["Runnable"]) -> List["Runnable"]: + return [r for r in runnables if r.type == RunnableType.galaxy_workflow and r.has_path] + + +def get_tool_ids_for_runnables(runnables) -> List[str]: + tool_ids = [] + for r in get_workflow_runnables(runnables): + tool_ids.extend(find_tool_ids(r.path)) + return list(dict.fromkeys(tool_ids)) + + def _shared_galaxy_properties(config_directory, kwds, for_tests): """Setup properties useful for local and Docker Galaxy instances. @@ -1201,6 +1221,27 @@ def _search_tool_path_for(path, target, extra_paths=None): return None +def get_tool_sheds_conf_for_runnables(runnables: Optional[List["Runnable"]]) -> Optional[str]: + if runnables: + tool_ids = get_tool_ids_for_runnables(runnables) + return get_shed_tools_conf_string_for_tool_ids(tool_ids) + return None + + +def get_shed_tools_conf_string_for_tool_ids(tool_ids: List[str]) -> str: + tool_shed_urls = set(get_toolshed_url_for_tool_id(tool_id) for tool_id in tool_ids if tool_id) + # always add main toolshed + tool_shed_urls.add("https://toolshed.g2.bx.psu.edu") + cleaned_tool_shed_urls = set(_ for _ in tool_shed_urls if _ is not None) + TOOL_SHEDS_CONF_TEMPLATE = Template("""${tool_shed_lines}""") + tool_sheds: List[str] = [] + # sort tool_shed_urls from shortest to longest, as https://github.com/galaxyproject/galaxy/blob/c7cb47a1b18ccd5b39075a705bbd2f34572755fe/lib/galaxy/util/tool_shed/tool_shed_registry.py#L106-L118 + # has a bug where a toolshed that is an exact substring of another registered toolshed would wrongly be selected. + for shed_url in sorted(cleaned_tool_shed_urls, key=lambda url: len(url)): + tool_sheds.append(f'') + return TOOL_SHEDS_CONF_TEMPLATE.substitute(tool_shed_lines="".join(tool_sheds)) + + def _configure_sheds_config_file(ctx, config_directory, **kwds): contents = kwds.get("tool_sheds_config_content") if not contents: diff --git a/planemo/galaxy/workflows.py b/planemo/galaxy/workflows.py index 9ad6e9abb..48ee9d135 100644 --- a/planemo/galaxy/workflows.py +++ b/planemo/galaxy/workflows.py @@ -61,6 +61,14 @@ def guess_tool_shed_url(tool_shed_fqdn: str) -> Optional[str]: return None +def get_toolshed_url_for_tool_id(tool_id: str) -> Optional[str]: + components = tool_id.split("/repos") + if len(components) > 1: + tool_shed_fqdn = components[0] + return guess_tool_shed_url(tool_shed_fqdn=tool_shed_fqdn) + return None + + def load_shed_repos(runnable): if runnable.type.name != "galaxy_workflow": return [] @@ -164,20 +172,25 @@ def _raw_dict(path, importer=None): return workflow -def find_tool_ids(path): - tool_ids = set() - workflow = _raw_dict(path) - - def register_tool_ids(tool_ids, workflow): - for step in workflow["steps"].values(): - if step.get("subworkflow"): - register_tool_ids(tool_ids, step["subworkflow"]) - elif step.get("tool_id"): - tool_ids.add(step["tool_id"]) +def get_tool_ids_for_workflow(wf_dict: Dict[str, Any], tool_ids: Optional[List[str]] = None) -> List[str]: + tool_ids = [] if tool_ids is None else tool_ids + steps = wf_dict["steps"].values() if isinstance(wf_dict["steps"], dict) else wf_dict["steps"] + for step in steps: + if step.get("type", "tool") == "tool" and not step.get("run", {}).get("class") == "GalaxyWorkflow": + tool_id = step["tool_id"] + tool_ids.append(tool_id) + elif step.get("type") == "subworkflow": # GA SWF + get_tool_ids_for_workflow(step["subworkflow"], tool_ids=tool_ids) + elif step.get("run", {}).get("class") == "GalaxyWorkflow": # gxformat2 SWF + get_tool_ids_for_workflow(step["run"], tool_ids=tool_ids) + else: + continue + return list(dict.fromkeys(tool_ids)) - register_tool_ids(tool_ids, workflow) - return list(tool_ids) +def find_tool_ids(path): + workflow = _raw_dict(path) + return get_tool_ids_for_workflow(workflow) WorkflowOutput = namedtuple("WorkflowOutput", ["order_index", "output_name", "label", "optional"]) diff --git a/tests/test_autoupdate.py b/tests/test_autoupdate.py index 005244ed6..b4a108f90 100644 --- a/tests/test_autoupdate.py +++ b/tests/test_autoupdate.py @@ -3,9 +3,9 @@ from galaxy.util import parse_xml_string -from planemo.autoupdate import ( - get_newest_tool_id, - get_shed_tools_conf_string_for_tool_ids, +from planemo.autoupdate import get_newest_tool_id +from planemo.galaxy.config import get_shed_tools_conf_string_for_tool_ids +from planemo.galaxy.workflows import ( get_tool_ids_for_workflow, get_toolshed_url_for_tool_id, ) From 0f6bb51431bd71986dd6f6cdceaa1187c7b6ca43 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 7 May 2024 17:23:14 +0200 Subject: [PATCH 13/14] Use MAIN_TOOLSHED_URL instead of literal url --- planemo/galaxy/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 98f518a70..144907d25 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -64,6 +64,7 @@ find_tool_ids, import_workflow, install_shed_repos, + MAIN_TOOLSHED_URL, ) if TYPE_CHECKING: @@ -1231,7 +1232,7 @@ def get_tool_sheds_conf_for_runnables(runnables: Optional[List["Runnable"]]) -> def get_shed_tools_conf_string_for_tool_ids(tool_ids: List[str]) -> str: tool_shed_urls = set(get_toolshed_url_for_tool_id(tool_id) for tool_id in tool_ids if tool_id) # always add main toolshed - tool_shed_urls.add("https://toolshed.g2.bx.psu.edu") + tool_shed_urls.add(MAIN_TOOLSHED_URL) cleaned_tool_shed_urls = set(_ for _ in tool_shed_urls if _ is not None) TOOL_SHEDS_CONF_TEMPLATE = Template("""${tool_shed_lines}""") tool_sheds: List[str] = [] From 6ad6e526ddf38539e24543a658b9f4c9b5036e24 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 7 May 2024 17:35:27 +0200 Subject: [PATCH 14/14] Simplify local_galaxy_config --- planemo/galaxy/config.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 144907d25..625210c22 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -263,11 +263,7 @@ def config_join(*args): shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools") _ensure_directory(shed_tool_path) - # Find tool sheds to add to config - tool_sheds_config_content = get_tool_sheds_conf_for_runnables(runnables) - if tool_sheds_config_content: - kwds["tool_sheds_config_content"] = tool_sheds_config_content - sheds_config_path = _configure_sheds_config_file(ctx, config_directory, **kwds) + sheds_config_path = _configure_sheds_config_file(ctx, config_directory, runnables, **kwds) port = _get_port(kwds) properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests) _handle_container_resolution(ctx, kwds, properties) @@ -398,11 +394,7 @@ def config_join(*args): shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools") _ensure_directory(shed_tool_path) - # Find tool sheds to add to config - tool_sheds_config_content = get_tool_sheds_conf_for_runnables(runnables) - if tool_sheds_config_content: - kwds["tool_sheds_config_content"] = tool_sheds_config_content - sheds_config_path = _configure_sheds_config_file(ctx, config_directory, **kwds) + sheds_config_path = _configure_sheds_config_file(ctx, config_directory, runnables, **kwds) database_location = config_join("galaxy.sqlite") master_api_key = _get_master_api_key(kwds) @@ -1243,8 +1235,9 @@ def get_shed_tools_conf_string_for_tool_ids(tool_ids: List[str]) -> str: return TOOL_SHEDS_CONF_TEMPLATE.substitute(tool_shed_lines="".join(tool_sheds)) -def _configure_sheds_config_file(ctx, config_directory, **kwds): - contents = kwds.get("tool_sheds_config_content") +def _configure_sheds_config_file(ctx, config_directory, runnables, **kwds): + # Find tool sheds to add to config + contents = get_tool_sheds_conf_for_runnables(runnables) if not contents: if "shed_target" not in kwds: kwds = kwds.copy()