Merge pull request #1399 from mvdbeek/fix_zenodo_tests

Fix zenodo test, drop explicit datatype mapping

docs/commands/shed_init.rst

@@ -29,7 +29,7 @@ to manage repositories in a Galaxy Tool Shed.
       --remote_repository_url TEXT    Specify repository remote_repository_url for
                                       .shed.yml.
       --homepage_url TEXT             Specify repository homepage_url for .shed.yml.
-      --category [Assembly|ChIP-seq|Climate Analysis|Combinatorial Selections|Computational chemistry|Constructive Solid Geometry|Convert Formats|Data Export|Data Managers|Data Source|Ecology|Entomology|Epigenetics|Fasta Manipulation|Fastq Manipulation|Flow Cytometry Analysis|Genome annotation|Genome editing|Genome-Wide Association Study|Genomic Interval Operations|GIS|Graphics|Imaging|InteractiveTools|Machine Learning|Metabolomics|Metagenomics|Micro-array Analysis|Molecular Dynamics|Nanopore|Next Gen Mappers|NLP|Ontology Manipulation|Phylogenetics|Proteomics|RNA|SAM|Sequence Analysis|Statistics|Structural Materials Analysis|Synthetic Biology|Systems Biology|Text Manipulation|Tool Dependency Packages|Tool Generators|Transcriptomics|Variant Analysis|Visualization|Web Services]
+      --category [Assembly|Astronomy|ChIP-seq|Climate Analysis|CLIP-seq|Combinatorial Selections|Computational chemistry|Constructive Solid Geometry|Convert Formats|Data Export|Data Managers|Data Source|Ecology|Entomology|Epigenetics|Fasta Manipulation|Fastq Manipulation|Flow Cytometry Analysis|Genome annotation|Genome editing|Genome-Wide Association Study|Genomic Interval Operations|GIS|Graphics|Imaging|InteractiveTools|Machine Learning|Materials science|Metabolomics|Metagenomics|Micro-array Analysis|Molecular Dynamics|Nanopore|Next Gen Mappers|NLP|Ontology Manipulation|Phylogenetics|Proteomics|RNA|SAM|Sequence Analysis|Statistics|Structural Materials Analysis|Synthetic Biology|Systems Biology|Text Manipulation|Tool Dependency Packages|Tool Generators|Transcriptomics|Variant Analysis|Visualization|Web Services]
                                       Specify repository category for .shed.yml (may
                                       specify multiple).
       --owner TEXT                    Tool Shed repository owner (username).

docs/commands/training_fill_data_library.rst

@@ -21,7 +21,5 @@ Build training template from workflow.
       --tutorial_name TEXT  Name (directory name) of the tutorial to modify
                             [required]
       --zenodo_link TEXT    Zenodo URL with the input data
-      --datatypes PATH      YAML file with the correspondance between Zenodo
-                            extension and Galaxy datatypes
       --help                Show this message and exit.
     

docs/commands/training_init.rst

@@ -34,8 +34,6 @@ Build training template from workflow.
                                       workflow
       --workflow_id TEXT              ID of the workflow on the Galaxy instance
       --zenodo_link TEXT              Zenodo URL with the input data
-      --datatypes PATH                YAML file with the correspondance between
-                                      Zenodo extension and Galaxy datatypes
       --galaxy_root DIRECTORY         Root of development galaxy directory to
                                       execute command with.
       --galaxy_python_version [3|3.7|3.8|3.9|3.10|3.11]

planemo/options.py

@@ -1283,15 +1283,6 @@ def training_tutorial_name_req_option():
     return planemo_option("--tutorial_name", required=True, help="Name (directory name) of the tutorial to modify")
 
 
-def training_datatype_option():
-    return planemo_option(
-        "--datatypes",
-        type=click.Path(file_okay=True, resolve_path=True),
-        help="YAML file with the correspondance between Zenodo extension and Galaxy datatypes",
-        default="shared/datatypes.yaml",
-    )
-
-
 def training_zenodo_option():
     return planemo_option("--zenodo_link", help="Zenodo URL with the input data")
 
@@ -1322,15 +1313,14 @@ def training_tutorial_option():
 
 
 def training_init_options():
-    return _compose(training_topic_option(), training_tutorial_option(), training_datatype_option())
+    return _compose(training_topic_option(), training_tutorial_option())
 
 
 def training_fill_data_library_options():
     return _compose(
         training_topic_name_option(),
         training_tutorial_name_req_option(),
         training_zenodo_option(),
-        training_datatype_option(),
     )
 
 

planemo/training/__init__.py

@@ -49,8 +49,6 @@ def check_topic_init_tuto(self):
         # initiate the tutorial
         self.tuto = Tutorial(training=self, topic=self.topic)
         self.tuto.init_from_existing_tutorial(self.kwds["tutorial_name"])
-        if "datatypes" in self.kwds:
-            self.tuto.datatype_fp = self.kwds["datatypes"]
         if "workflow" in self.kwds:
             self.tuto.init_wf_fp = self.kwds["workflow"]
         if "workflow_id" in self.kwds:

planemo/training/tutorial.py

@@ -279,7 +279,6 @@ def __init__(self, training, topic, name="new_tuto", title="The new tutorial", z
         self.init_wf_id = None
         self.hands_on = True
         self.slides = False
-        self.datatype_fp = ""
         self.set_dir_name()
         self.init_data_lib()
         self.body = templates.render(
@@ -309,7 +308,6 @@ def init_from_kwds(self, kwds):
         self.init_wf_id = kwds["workflow_id"]
         self.hands_on = kwds["hands_on"]
         self.slides = kwds["slides"]
-        self.datatype_fp = kwds["datatypes"]
         self.set_dir_name()
         self.init_data_lib()
 
@@ -443,9 +441,7 @@ def get_files_from_zenodo(self):
 
         files = []
         for f in req_res["files"]:
-            file_dict = {"url": "", "src": "url", "ext": "", "info": self.zenodo_link}
-            if "type" in f:
-                file_dict["ext"] = get_galaxy_datatype(f["type"], self.datatype_fp)
+            file_dict = {"url": "", "src": "url", "ext": "auto", "info": self.zenodo_link}
             if "links" not in f and "self" not in f["links"]:
                 raise ValueError("No link for file %s" % f)
             file_dict["url"] = f["links"]["self"]
@@ -546,18 +542,6 @@ def create_tutorial(self, ctx):
                 handle.write("---\nlayout: faq-page\n---\n")
 
 
-def get_galaxy_datatype(z_ext, datatype_fp):
-    """Get the Galaxy datatype corresponding to a Zenodo file type."""
-    g_datatype = ""
-    datatypes = load_yaml(datatype_fp)
-    if z_ext in datatypes:
-        g_datatype = datatypes[z_ext]
-    if g_datatype == "":
-        g_datatype = "# Please add a Galaxy datatype or update the shared/datatypes.yaml file"
-    info(f"Get Galaxy datatypes: {z_ext} --> {g_datatype}")
-    return g_datatype
-
-
 def get_zenodo_record(zenodo_link):
     """Get the content of a Zenodo record."""
     # get the record in the Zenodo link

tests/data/wf7-collection-output.gxwf-test.yml

@@ -14,4 +14,4 @@
         el1:
           elements:
             forward:
-              checksum: "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
+              checksum: "sha1$a0b65939670bc2c010f4d5d6a0b3e4e4590fb92b"

tests/test_cmd_training_fill_data_library.py

@@ -1,12 +1,6 @@
 """Tests for the ``training_fill_data_library`` command."""
-import os
-import shutil
-
 from .test_cmd_training_generate_from_wf import create_tutorial_dir
-from .test_utils import (
-    CliTestCase,
-    TEST_DATA_DIR,
-)
+from .test_utils import CliTestCase
 
 
 class CmdTrainingFillDataLibraryTestCase(CliTestCase):
@@ -29,20 +23,6 @@ def test_training_fill_data_library_command_tutorial_topic(self):
         with self._isolate():
             topic_n = "test"
             tuto_n = "test"
-            datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
-            # not working
-            create_tutorial_dir(topic_n, tuto_n)
-            training_fill_data_library_command = [
-                "training_fill_data_library",
-                "--topic_name",
-                topic_n,
-                "--tutorial_name",
-                tuto_n,
-                "--datatypes",
-                datatype,
-            ]
-            shutil.rmtree("topics")
-            self._check_exit_code(training_fill_data_library_command, exit_code=1)
             # working
             create_tutorial_dir(topic_n, tuto_n)
             training_fill_data_library_command = [
@@ -51,8 +31,6 @@ def test_training_fill_data_library_command_tutorial_topic(self):
                 topic_n,
                 "--tutorial_name",
                 tuto_n,
-                "--datatypes",
-                datatype,
             ]
             self._check_exit_code(training_fill_data_library_command, exit_code=0)
 
@@ -62,19 +40,6 @@ def test_training_fill_data_library_command_tutorial_zenodo(self):
             topic_n = "test"
             tuto_n = "test"
             create_tutorial_dir(topic_n, tuto_n)
-            datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
-            # not working test
-            training_fill_data_library_command = [
-                "training_fill_data_library",
-                "--topic_name",
-                topic_n,
-                "--tutorial_name",
-                tuto_n,
-                "--zenodo_link",
-                "https://zenodo.org/record/1321885",
-            ]
-            self._check_exit_code(training_fill_data_library_command, exit_code=1)
-            # working
             training_fill_data_library_command = [
                 "training_fill_data_library",
                 "--topic_name",
@@ -83,7 +48,5 @@ def test_training_fill_data_library_command_tutorial_zenodo(self):
                 tuto_n,
                 "--zenodo_link",
                 "https://zenodo.org/record/1321885",
-                "--datatypes",
-                datatype,
             ]
             self._check_exit_code(training_fill_data_library_command, exit_code=0)

tests/test_cmd_training_init.py

@@ -79,18 +79,6 @@ def test_training_init_command_tutorial(self):
     def test_training_init_command_tutorial_zenodo(self):
         """Test training_init command to create new tutorial with zenodo."""
         with self._isolate():
-            datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
-            # not working test
-            training_init_command = [
-                "training_init",
-                "--topic_name",
-                "test",
-                "--tutorial_name",
-                "test",
-                "--zenodo_link",
-                "https://zenodo.org/record/1321885",
-            ]
-            self._check_exit_code(training_init_command, exit_code=1)
             # working
             training_init_command = [
                 "training_init",
@@ -100,8 +88,6 @@ def test_training_init_command_tutorial_zenodo(self):
                 "test",
                 "--zenodo_link",
                 "https://zenodo.org/record/1321885",
-                "--datatypes",
-                datatype,
             ]
             self._check_exit_code(training_init_command, exit_code=0)
 

tests/test_training.py

@@ -13,7 +13,6 @@
     TEST_DATA_DIR,
 )
 
-datatype_fp = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
 tuto_fp = os.path.join(TEST_DATA_DIR, "training_tutorial.md")
 tuto_wo_zenodo_fp = os.path.join(TEST_DATA_DIR, "training_tutorial_wo_zenodo.md")
 zenodo_link = "https://zenodo.org/record/1321885"
@@ -40,7 +39,6 @@
     "workflow": None,
     "workflow_id": None,
     "zenodo_link": None,
-    "datatypes": os.path.join(TEST_DATA_DIR, "training_datatypes.yaml"),
     "templates": None,
     # planemo configuation
     "conda_auto_init": True,
@@ -175,7 +173,6 @@ def test_training_check_topic_init_tuto() -> None:
     create_existing_tutorial("existing_tutorial", tuto_fp, train.topic)
     train.check_topic_init_tuto()
     assert train.tuto.name == train.kwds["tutorial_name"]
-    assert train.tuto.datatype_fp
     # clean after
     shutil.rmtree(train.topics_dir)
     shutil.rmtree("metadata")

tests/test_training_tutorial.py

@@ -9,7 +9,6 @@
 from planemo.training import Training
 from planemo.training.topic import Topic
 from planemo.training.tutorial import (
-    get_galaxy_datatype,
     get_hands_on_boxes_from_local_galaxy,
     get_hands_on_boxes_from_running_galaxy,
     get_wf_inputs,
@@ -21,7 +20,6 @@
 from .test_training import (
     create_existing_tutorial,
     CTX,
-    datatype_fp,
     KWDS,
     RUNNABLE,
     tuto_fp,
@@ -36,20 +34,14 @@
 training = Training(KWDS)
 
 
-def test_get_galaxy_datatype() -> None:
-    """Test :func:`planemo.training.tutorial.get_galaxy_datatype`."""
-    assert get_galaxy_datatype("csv", datatype_fp) == "csv"
-    assert get_galaxy_datatype("test", datatype_fp) == "strange_datatype"
-    assert "# Please add" in get_galaxy_datatype("unknown", datatype_fp)
-
-
 def test_get_zenodo_record() -> None:
     """Test :func:`planemo.training.tutorial.get_zenodo_record`."""
     z_record, req_res = get_zenodo_record(zenodo_link)
-    file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f"
+    file_link_prefix = f"https://zenodo.org/api/records/{z_record}"
     assert z_record == "1321885"
     assert "files" in req_res
-    assert req_res["files"][0]["type"] in ["rdata", "csv"]
+    print(req_res["files"][0])
+    assert req_res["files"][0]["key"].endswith(".RData")
     assert file_link_prefix in req_res["files"][0]["links"]["self"]
     # check with wrong zenodo link
     z_record, req_res = get_zenodo_record("https://zenodo.org/api/records/zenodooo")
@@ -59,10 +51,10 @@ def test_get_zenodo_record() -> None:
     # using DOI
     z_link = "https://doi.org/10.5281/zenodo.1321885"
     z_record, req_res = get_zenodo_record(z_link)
-    file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f"
+    file_link_prefix = f"https://zenodo.org/api/records/{z_record}"
     assert z_record == "1321885"
     assert "files" in req_res
-    assert req_res["files"][0]["type"] in ["rdata", "csv"]
+    assert req_res["files"][0]["key"].endswith(".RData")
     assert file_link_prefix in req_res["files"][0]["links"]["self"]
 
 
@@ -135,7 +127,6 @@ def test_tutorial_init() -> None:
     assert not tuto.slides
     assert tuto.init_wf_id is None
     assert tuto.init_wf_fp is None
-    assert tuto.datatype_fp == ""
     assert "new_tuto" in tuto.dir
     assert "## Sub-step with **My Tool**" in tuto.body
     assert tuto.data_lib
@@ -157,7 +148,6 @@ def test_tutorial_init_from_kwds() -> None:
         "workflow": WF_FP,
         "workflow_id": "id",
         "zenodo_link": None,
-        "datatypes": datatype_fp,
     }
     tuto = Tutorial(training=training, topic=topic)
     tuto.init_from_kwds(kwds)
@@ -169,7 +159,6 @@ def test_tutorial_init_from_kwds() -> None:
     assert tuto.slides
     assert tuto.init_wf_id == "id"
     assert tuto.init_wf_fp == WF_FP
-    assert tuto.datatype_fp == datatype_fp
     assert "my_tuto" in tuto.dir
 
 
@@ -289,25 +278,21 @@ def test_tutorial_export_workflow_file() -> None:
 def test_tutorial_get_files_from_zenodo() -> None:
     """Test :func:`planemo.training.tutorial.tutorial.get_files_from_zenodo`."""
     tuto = Tutorial(training=training, topic=topic, zenodo_link=zenodo_link)
-    tuto.datatype_fp = datatype_fp
     files, z_record = tuto.get_files_from_zenodo()
     assert z_record == "1321885"
     # test links
-    file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f"
-    assert file_link_prefix in tuto.zenodo_file_links[0]
+    assert z_record in tuto.zenodo_file_links[0]
     # test files dict
-    assert file_link_prefix in files[0]["url"]
+    assert z_record in files[0]["url"]
     assert files[0]["src"] == "url"
     assert files[0]["info"] == zenodo_link
-    assert "# Please add" in files[0]["ext"]
-    assert files[1]["ext"] == "csv"
+    assert ".csv" in files[1]["url"]
 
 
 def test_tutorial_prepare_data_library_from_zenodo() -> None:
     """Test :func:`planemo.training.tutorial.tutorial.prepare_data_library_from_zenodo`."""
     # without zenodo link
     tuto = Tutorial(training=training, topic=topic)
-    tuto.datatype_fp = datatype_fp
     os.makedirs(tuto.wf_dir)
     tuto.prepare_data_library_from_zenodo()
     assert os.path.exists(tuto.data_lib_fp)
@@ -383,7 +368,6 @@ def test_tutorial_create_tutorial() -> None:
             "workflow": WF_FP,
             "workflow_id": None,
             "zenodo_link": zenodo_link,
-            "datatypes": datatype_fp,
         }
     )
     tuto.create_tutorial(CTX)