From 37dafed147019a6194dac562a9cc1a8221f7e182 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 6 Nov 2024 15:20:14 +0100 Subject: [PATCH 01/14] Change type from Extract name to library name --- test-data/biosamples-input-isa.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/biosamples-input-isa.json b/test-data/biosamples-input-isa.json index 4aaf6c9..6155570 100644 --- a/test-data/biosamples-input-isa.json +++ b/test-data/biosamples-input-isa.json @@ -613,7 +613,7 @@ { "@id": "#other_material/332", "name": "extract 1", - "type": "Extract Name", + "type": "library name", "characteristics": [], "derivesFrom": [ { @@ -624,7 +624,7 @@ { "@id": "#other_material/333", "name": "library 1", - "type": "Extract Name", + "type": "library name", "characteristics": [ { "category": { From ea7bf3c589d0b4a2e4b4ba40a09df0e27385dc5f Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Tue, 5 Nov 2024 13:16:45 +0100 Subject: [PATCH 02/14] Return updated ISA JSON instead of investigation --- mars-cli/mars_lib/isa_json.py | 11 ++++++----- mars-cli/tests/test_isa_json.py | 29 +++++++++++++++++------------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/mars-cli/mars_lib/isa_json.py b/mars-cli/mars_lib/isa_json.py index 8507d1a..a0217e2 100644 --- a/mars-cli/mars_lib/isa_json.py +++ b/mars-cli/mars_lib/isa_json.py @@ -353,8 +353,8 @@ def create_accession_characteristic( def update_investigation( - investigation: Investigation, repo_response: RepositoryResponse -) -> Investigation: + isa_json: IsaJson, repo_response: RepositoryResponse +) -> IsaJson: """ Adds the accession to the ISA JSON. @@ -365,7 +365,7 @@ def update_investigation( Returns: Investigation: The updated ISA JSON. """ - updated_investigation = investigation.model_copy(deep=True) + investigation = isa_json.investigation for accession in repo_response.accessions: has_assay_in_path = [p for p in accession.path if p.key == "assays"] @@ -380,7 +380,7 @@ def update_investigation( if not study_filter: raise ValueError(f"Study filter is not present in {accession.path}.") - updated_node = apply_filter(study_filter, updated_investigation.studies) + updated_node = apply_filter(study_filter, investigation.studies) if target_level == "assay": assay_filter = get_filter_for_accession_key(accession, "assays") @@ -407,4 +407,5 @@ def update_investigation( add_accession_to_node(updated_node, accession.value, material_type_path) - return updated_investigation + isa_json.investigation = investigation + return isa_json diff --git a/mars-cli/tests/test_isa_json.py b/mars-cli/tests/test_isa_json.py index f14f9ba..88e5c92 100644 --- a/mars-cli/tests/test_isa_json.py +++ b/mars-cli/tests/test_isa_json.py @@ -186,21 +186,25 @@ def test_update_study_materials_no_accession_categories(): respose_file_path = "tests/fixtures/json_responses/biosamples_success_reponse.json" repo_response = RepositoryResponse.from_json_file(respose_file_path) - updated_investigation = update_investigation( - validated_isa_json.investigation, repo_response - ) + updated_isa_json = update_investigation(validated_isa_json, repo_response) # Check the accession number of the source # Accession characteristic is of type String assert ( - updated_investigation.studies[0].materials.sources[0].characteristics[-1].value + updated_isa_json.investigation.studies[0] + .materials.sources[0] + .characteristics[-1] + .value == repo_response.accessions[0].value ) # Check the accession number of the sample # Accession characteristic is of type String assert ( - updated_investigation.studies[0].materials.samples[0].characteristics[-1].value + updated_isa_json.investigation.studies[0] + .materials.samples[0] + .characteristics[-1] + .value == repo_response.accessions[1].value ) @@ -213,16 +217,14 @@ def test_update_study_materials_with_accession_categories(): validated_isa_json = IsaJson.model_validate(json_data) - respose_file_path = "tests/fixtures/json_responses/biosamples_success_reponse.json" - repo_response = RepositoryResponse.from_json_file(respose_file_path) + response_file_path = "tests/fixtures/json_responses/biosamples_success_reponse.json" + repo_response = RepositoryResponse.from_json_file(response_file_path) - updated_investigation = update_investigation( - validated_isa_json.investigation, repo_response - ) + updated_isa_json = update_investigation(validated_isa_json, repo_response) # Check the accession number of the source # Accession characteristic is of type OntologyAnnotation assert ( - updated_investigation.studies[0] + updated_isa_json.investigation.studies[0] .materials.sources[0] .characteristics[-1] .value.annotationValue @@ -232,7 +234,10 @@ def test_update_study_materials_with_accession_categories(): # Check the accession number of the sample # Accession characteristic is of type String assert ( - updated_investigation.studies[0].materials.samples[0].characteristics[-1].value + updated_isa_json.investigation.studies[0] + .materials.samples[0] + .characteristics[-1] + .value == repo_response.accessions[1].value ) From 3ac4abf99b6c664f7184f566f5d171971dd085b6 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Tue, 5 Nov 2024 13:23:32 +0100 Subject: [PATCH 03/14] Return filtered ISA JSON instead of investigation --- mars-cli/mars_lib/isa_json.py | 20 +++++++++----------- mars-cli/tests/test_isa_json.py | 14 +++++++------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/mars-cli/mars_lib/isa_json.py b/mars-cli/mars_lib/isa_json.py index a0217e2..479be92 100644 --- a/mars-cli/mars_lib/isa_json.py +++ b/mars-cli/mars_lib/isa_json.py @@ -22,21 +22,21 @@ def reduce_isa_json_for_target_repo( - input_isa_json: Investigation, target_repo: str -) -> Investigation: + input_isa_json: IsaJson, target_repo: str +) -> IsaJson: """ Filters out assays that are not meant to be sent to the specified target repository. Args: - input_isa_json (Investigation): Input ISA JSON that contains the original information. + input_isa_json (IsaJson): Input ISA JSON that contains the original information. target_repo (TargetRepository): Target repository as a constant. Returns: - Investigation: Filtered ISA JSON. + IsaJson: Filtered ISA JSON. """ filtered_isa_json = input_isa_json.model_copy(deep=True) new_studies = [] - studies = filtered_isa_json.studies + studies = filtered_isa_json.investigation.studies for study in studies: if target_repo == TargetRepository.BIOSAMPLES: filtered_assays = [] @@ -51,7 +51,7 @@ def reduce_isa_json_for_target_repo( study.assays = filtered_assays new_studies.append(study) - filtered_isa_json.studies = new_studies + filtered_isa_json.investigation.studies = new_studies return filtered_isa_json @@ -352,18 +352,16 @@ def create_accession_characteristic( updated_material.characteristics.append(new_material_attribute_value) -def update_investigation( - isa_json: IsaJson, repo_response: RepositoryResponse -) -> IsaJson: +def update_isa_json(isa_json: IsaJson, repo_response: RepositoryResponse) -> IsaJson: """ Adds the accession to the ISA JSON. Args: - isa_json (Investigation): The ISA JSON to be updated. + isa_json (IsaJson): The ISA JSON to be updated. repo_response (RepositoryResponse): The response from the repository. Returns: - Investigation: The updated ISA JSON. + IsaJson: The updated ISA JSON. """ investigation = isa_json.investigation for accession in repo_response.accessions: diff --git a/mars-cli/tests/test_isa_json.py b/mars-cli/tests/test_isa_json.py index 88e5c92..25270f4 100644 --- a/mars-cli/tests/test_isa_json.py +++ b/mars-cli/tests/test_isa_json.py @@ -3,7 +3,7 @@ from mars_lib.isa_json import ( reduce_isa_json_for_target_repo, load_isa_json, - update_investigation, + update_isa_json, ) from mars_lib.target_repo import TargetRepository, TARGET_REPO_KEY import pytest @@ -44,12 +44,12 @@ def test_reduce_isa_json_for_target_repo(): ) filtered_isa_json = reduce_isa_json_for_target_repo( - good_isa_json.investigation, TargetRepository.ENA + good_isa_json, TargetRepository.ENA ) good_isa_json_study = good_isa_json.investigation.studies[0] - filtered_isa_json_study = filtered_isa_json.studies[0] + filtered_isa_json_study = filtered_isa_json.investigation.studies[0] assert len(good_isa_json_study.assays) == 5 assert len(filtered_isa_json_study.assays) == 1 @@ -61,10 +61,10 @@ def test_reduce_isa_json_for_biosamples(): ) filtered_isa_json = reduce_isa_json_for_target_repo( - good_isa_json.investigation, TargetRepository.BIOSAMPLES + good_isa_json, TargetRepository.BIOSAMPLES ) - assert len(filtered_isa_json.studies[0].assays) == 0 + assert len(filtered_isa_json.investigation.studies[0].assays) == 0 def test_data_type_validator(): @@ -186,7 +186,7 @@ def test_update_study_materials_no_accession_categories(): respose_file_path = "tests/fixtures/json_responses/biosamples_success_reponse.json" repo_response = RepositoryResponse.from_json_file(respose_file_path) - updated_isa_json = update_investigation(validated_isa_json, repo_response) + updated_isa_json = update_isa_json(validated_isa_json, repo_response) # Check the accession number of the source # Accession characteristic is of type String @@ -220,7 +220,7 @@ def test_update_study_materials_with_accession_categories(): response_file_path = "tests/fixtures/json_responses/biosamples_success_reponse.json" repo_response = RepositoryResponse.from_json_file(response_file_path) - updated_isa_json = update_investigation(validated_isa_json, repo_response) + updated_isa_json = update_isa_json(validated_isa_json, repo_response) # Check the accession number of the source # Accession characteristic is of type OntologyAnnotation assert ( From f718d26937b50caee6436fbd3e50597ac8aee065 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Tue, 5 Nov 2024 13:28:51 +0100 Subject: [PATCH 04/14] Add filtering of the ISA JSON just before submission --- mars-cli/mars_lib/submit.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mars-cli/mars_lib/submit.py b/mars-cli/mars_lib/submit.py index a6d16dc..8ffbe15 100644 --- a/mars-cli/mars_lib/submit.py +++ b/mars-cli/mars_lib/submit.py @@ -11,7 +11,7 @@ input_json_schema_filepath, ) from mars_lib.credential import CredentialManager -from mars_lib.isa_json import load_isa_json +from mars_lib.isa_json import load_isa_json, reduce_isa_json_for_target_repo from mars_lib.models.isa_json import IsaJson from mars_lib.target_repo import TargetRepository from mars_lib.logging import print_and_log @@ -54,7 +54,6 @@ def submission( ) if TargetRepository.ENA in target_repositories: - # TODO: Filter out other assays ena_result = submit_to_ena( isa_json=isa_json, user_credentials=user_credentials, @@ -114,7 +113,9 @@ def submit_to_biosamples( biosamples_url, headers=headers, params=params, - json=isa_json.model_dump(by_alias=True, exclude_none=True), + json=reduce_isa_json_for_target_repo( + isa_json, TargetRepository.BIOSAMPLES + ).model_dump(by_alias=True, exclude_none=True), ) if result.status_code != 200: @@ -142,7 +143,9 @@ def submit_to_ena( submission_url, headers=headers, params=params, - json=isa_json.model_dump(by_alias=True, exclude_none=True), + json=reduce_isa_json_for_target_repo(isa_json, TargetRepository.ENA).model_dump( + by_alias=True, exclude_none=True + ), ) if result.status_code != 200: From 73d3cde4d0133f06841f2b94fa5dd1345b712557 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Tue, 5 Nov 2024 15:16:52 +0100 Subject: [PATCH 05/14] Add submit to biosamples option --- mars-cli/mars_cli.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/mars-cli/mars_cli.py b/mars-cli/mars_cli.py index 7fb07ad..2c5bf3e 100644 --- a/mars-cli/mars_cli.py +++ b/mars-cli/mars_cli.py @@ -166,6 +166,12 @@ def cli(ctx, development): help="Name of a credentials file", ) @click.argument("isa_json_file", type=click.File("r")) +@click.option( + "--submit-to-biosamples", + type=click.BOOL, + default=True, + help="Submit to BioSamples.", +) @click.option("--submit-to-ena", type=click.BOOL, default=True, help="Submit to ENA.") @click.option( "--submit-to-metabolights", @@ -186,6 +192,7 @@ def submit( username_credentials, credentials_file, isa_json_file, + submit_to_biosamples, submit_to_ena, submit_to_metabolights, investigation_is_root, @@ -193,13 +200,11 @@ def submit( """Start a submission to the target repositories.""" target_repositories = [TargetRepository.BIOSAMPLES] + if submit_to_biosamples: + target_repositories.append(TargetRepository.BIOSAMPLES) + if submit_to_ena: target_repositories.append(TargetRepository.ENA) - target_repositories.remove(TargetRepository.BIOSAMPLES) - print_and_log( - f"Skipping {TargetRepository.BIOSAMPLES} repository due to {TargetRepository.ENA} being present in the list of repositories", - level="debug", - ) if submit_to_metabolights: target_repositories.append(TargetRepository.METABOLIGHTS) From af49bfa0bfd64a56a8218543a606b2a432c287ec Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Tue, 5 Nov 2024 15:17:18 +0100 Subject: [PATCH 06/14] Add class method for all available repositories --- mars-cli/mars_lib/target_repo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mars-cli/mars_lib/target_repo.py b/mars-cli/mars_lib/target_repo.py index ef6e3f6..dd648ec 100644 --- a/mars-cli/mars_lib/target_repo.py +++ b/mars-cli/mars_lib/target_repo.py @@ -13,3 +13,7 @@ class TargetRepository(str, Enum): METABOLIGHTS = "metabolights" BIOSAMPLES = "biosamples" EVA = "eva" + + @classmethod + def available_repositories(cls): + return {item.value for item in cls} From 7c3787229e737c2bc7debe71f8079970511c08f5 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Tue, 5 Nov 2024 18:11:21 +0100 Subject: [PATCH 07/14] Add more debugging functionality --- mars-cli/mars_cli.py | 2 +- mars-cli/mars_lib/isa_json.py | 2 + mars-cli/mars_lib/submit.py | 73 +++++++++++++++++++++++++++-------- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/mars-cli/mars_cli.py b/mars-cli/mars_cli.py index 2c5bf3e..85efb9d 100644 --- a/mars-cli/mars_cli.py +++ b/mars-cli/mars_cli.py @@ -198,7 +198,7 @@ def submit( investigation_is_root, ): """Start a submission to the target repositories.""" - target_repositories = [TargetRepository.BIOSAMPLES] + target_repositories = [] if submit_to_biosamples: target_repositories.append(TargetRepository.BIOSAMPLES) diff --git a/mars-cli/mars_lib/isa_json.py b/mars-cli/mars_lib/isa_json.py index 479be92..0a61b28 100644 --- a/mars-cli/mars_lib/isa_json.py +++ b/mars-cli/mars_lib/isa_json.py @@ -64,6 +64,8 @@ def detect_target_repo_comment(comments: List[Comment]) -> Comment: Returns: Comment: The comment where the name corresponds with the name of the provided target repo. """ + if len(comments) < 1: + raise ValueError("No comments found! Not able to detect the target repository!") return next(comment for comment in comments if comment.name == TARGET_REPO_KEY) diff --git a/mars-cli/mars_lib/submit.py b/mars-cli/mars_lib/submit.py index 8ffbe15..a936dc7 100644 --- a/mars-cli/mars_lib/submit.py +++ b/mars-cli/mars_lib/submit.py @@ -1,3 +1,5 @@ +import os +from datetime import datetime from io import TextIOWrapper import requests import json @@ -11,13 +13,29 @@ input_json_schema_filepath, ) from mars_lib.credential import CredentialManager -from mars_lib.isa_json import load_isa_json, reduce_isa_json_for_target_repo +from mars_lib.isa_json import ( + load_isa_json, + reduce_isa_json_for_target_repo, + update_isa_json, +) from mars_lib.models.isa_json import IsaJson +from mars_lib.models.repository_response import RepositoryResponse from mars_lib.target_repo import TargetRepository from mars_lib.logging import print_and_log from pydantic import ValidationError +def save_step_to_file(time_stamp: float, filename: str, isa_json: IsaJson): + dir_path = f"tmp/{str(time_stamp)}" + os.makedirs(dir_path, exist_ok=True) + + with open(f"{dir_path}/{filename}.json", "w") as f: + f.write(isa_json.model_dump_json(by_alias=True, exclude_none=True)) + + +DEBUG = os.getenv("MARS_DEBUG") in ["1", 1] + + def submission( credential_service_name: str, username_credentials: str, @@ -53,17 +71,18 @@ def submission( f"ISA JSON with investigation '{isa_json.investigation.title}' is valid." ) - if TargetRepository.ENA in target_repositories: - ena_result = submit_to_ena( - isa_json=isa_json, - user_credentials=user_credentials, - submission_url=urls["ENA"]["SUBMISSION"], - ) - print_and_log( - f"Submission to {TargetRepository.ENA} was successful. Result:\n{ena_result.json()}" - ) - # TODO: Update `isa_json`, based on the receipt returned - elif TargetRepository.BIOSAMPLES in target_repositories: + time_stamp = datetime.timestamp(datetime.now()) + + if DEBUG: + save_step_to_file(time_stamp, "0_Initial_ISA_JSON_in_model", isa_json) + + if all( + repo not in TargetRepository.available_repositories() + for repo in target_repositories + ): + raise ValueError("No target repository selected.") + + if TargetRepository.BIOSAMPLES in target_repositories: # Submit to Biosamples biosamples_result = submit_to_biosamples( isa_json=isa_json, @@ -76,7 +95,29 @@ def submission( level="info", ) # TODO: Update `isa_json`, based on the receipt returned - elif TargetRepository.METABOLIGHTS in target_repositories: + bs_mars_receipt = RepositoryResponse.model_validate( + json.loads(biosamples_result.content) + ) + isa_json = update_isa_json(isa_json, bs_mars_receipt) + if DEBUG: + save_step_to_file(time_stamp, "1_after_biosamples", isa_json) + + if TargetRepository.ENA in target_repositories: + ena_result = submit_to_ena( + isa_json=isa_json, + user_credentials=user_credentials, + submission_url=urls["ENA"]["SUBMISSION"], + ) + print_and_log( + f"Submission to {TargetRepository.ENA} was successful. Result:\n{ena_result.json()}" + ) + # TODO: Update `isa_json`, based on the receipt returned + ena_mars_receipt = RepositoryResponse.from_json(str(ena_result.content)) + isa_json = update_isa_json(isa_json, ena_mars_receipt) + if DEBUG: + save_step_to_file(time_stamp, "2_after_ena", isa_json) + + if TargetRepository.METABOLIGHTS in target_repositories: # Submit to MetaboLights # TODO: Filter out other assays print_and_log( @@ -84,17 +125,17 @@ def submission( level="info", ) # TODO: Update `isa_json`, based on the receipt returned - elif TargetRepository.EVA in target_repositories: + + if TargetRepository.EVA in target_repositories: # Submit to EVA # TODO: Filter out other assays print_and_log( f"Submission to {TargetRepository.EVA} was successful", level="info" ) # TODO: Update `isa_json`, based on the receipt returned - else: - raise ValueError("No target repository selected.") # TODO: Return the updated ISA JSON + return isa_json def submit_to_biosamples( From 53fc1777b09f53b8b8854a4d580ca67aa0bfbc79 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 6 Nov 2024 13:37:26 +0100 Subject: [PATCH 08/14] Fix finding accession characteristic --- .../WebinIsaToXmlSubmissionController.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java b/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java index b378a76..9088199 100644 --- a/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java +++ b/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java @@ -20,6 +20,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; import org.dom4j.Document; import org.dom4j.DocumentHelper; @@ -146,13 +148,20 @@ public Map getBiosamples(List studies) { } private String getCharacteresticAnnotation(List characteristics) { - for (Characteristic characteristic : characteristics) { - if ("#characteristic_category/accession".equals(characteristic.category.id)) { - return characteristic.value.annotationValue; - } + List filteredCharacteristics = characteristics.stream() + .filter(characteristic -> characteristic.category.id.contains("#characteristic_category/accession")) + .collect(Collectors.toList()); + + if (filteredCharacteristics.isEmpty()) { + log.error("No accession found in the characteristics"); + throw new RuntimeException("No accession found in the characteristics"); } - return ""; + if (filteredCharacteristics.size() > 1) { + log.error("More than one accession found in the characteristics"); + throw new RuntimeException("Too many accessions found in the characteristics"); + } + return filteredCharacteristics.get(0).value.annotationValue; } private static Element startPreparingWebinV2SubmissionXml(Document document) { From 41b56fb559c38658ee794a41ceec001879d7d1d1 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 6 Nov 2024 14:34:40 +0100 Subject: [PATCH 09/14] Modifiy find accession characteristic --- mars-cli/mars_lib/isa_json.py | 16 +++++++++------- mars-cli/mars_lib/submit.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/mars-cli/mars_lib/isa_json.py b/mars-cli/mars_lib/isa_json.py index 0a61b28..0f335b9 100644 --- a/mars-cli/mars_lib/isa_json.py +++ b/mars-cli/mars_lib/isa_json.py @@ -190,13 +190,15 @@ def accession_characteristic_present( f"'where' atribute is missing in path {material_type_path.key}." ) - accession_characteristics = [ - char - for char in material.characteristics - if char.category - and char.category.characteristicType - and char.category.characteristicType.annotationValue == "accession" - ] + accession_characteristics = [] + for char in material.characteristics: + if char.category and char.category.characteristicType: + if char.category.characteristicType.annotationValue: + if char.category.characteristicType.annotationValue == "accession": + accession_characteristics.append(char) + else: + if char.category.characteristicType == "accession": + accession_characteristics.append(char) if len(accession_characteristics) > 1: raise AttributeError( diff --git a/mars-cli/mars_lib/submit.py b/mars-cli/mars_lib/submit.py index a936dc7..7c6f5b4 100644 --- a/mars-cli/mars_lib/submit.py +++ b/mars-cli/mars_lib/submit.py @@ -94,7 +94,7 @@ def submission( f"Submission to {TargetRepository.BIOSAMPLES} was successful. Result:\n{biosamples_result.json()}", level="info", ) - # TODO: Update `isa_json`, based on the receipt returned + # Update `isa_json`, based on the receipt returned bs_mars_receipt = RepositoryResponse.model_validate( json.loads(biosamples_result.content) ) From 481bd07af89c28324034ff4e7b9f81df5ef1df15 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 6 Nov 2024 15:18:40 +0100 Subject: [PATCH 10/14] Always add accession characteristic annotationValue --- mars-cli/mars_lib/isa_json.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/mars-cli/mars_lib/isa_json.py b/mars-cli/mars_lib/isa_json.py index 0f335b9..fdff019 100644 --- a/mars-cli/mars_lib/isa_json.py +++ b/mars-cli/mars_lib/isa_json.py @@ -259,17 +259,12 @@ def add_accession_to_node( if not updated_material_accession_characteristic: raise ValueError("Accession characteristic is not present.") - if updated_material_accession_characteristic.value and hasattr( - updated_material_accession_characteristic.value, "annotationValue" - ): - accession_ontology_annotation = OntologyAnnotation() - accession_ontology_annotation.id = ( - f"#ontology_annotation/accession_{updated_material.id}" - ) - accession_ontology_annotation.annotationValue = accession_number - updated_material_accession_characteristic.value = accession_ontology_annotation - else: - updated_material_accession_characteristic.value = accession_number + accession_ontology_annotation = OntologyAnnotation() + accession_ontology_annotation.id = ( + f"#ontology_annotation/accession_{updated_material.id}" + ) + accession_ontology_annotation.annotationValue = accession_number + updated_material_accession_characteristic.value = accession_ontology_annotation updated_material.characteristics.append(updated_material_accession_characteristic) print(f"{updated_material.id}: {updated_material_accession_characteristic.value}.") From 6e4d0c6e57f5261fddfdd9ed0645fc22f634ae0d Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 6 Nov 2024 15:53:46 +0100 Subject: [PATCH 11/14] Fix failing tests --- mars-cli/mars_lib/submit.py | 2 +- mars-cli/tests/test_isa_json.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/mars-cli/mars_lib/submit.py b/mars-cli/mars_lib/submit.py index 7c6f5b4..106c301 100644 --- a/mars-cli/mars_lib/submit.py +++ b/mars-cli/mars_lib/submit.py @@ -111,7 +111,7 @@ def submission( print_and_log( f"Submission to {TargetRepository.ENA} was successful. Result:\n{ena_result.json()}" ) - # TODO: Update `isa_json`, based on the receipt returned + # Update `isa_json`, based on the receipt returned ena_mars_receipt = RepositoryResponse.from_json(str(ena_result.content)) isa_json = update_isa_json(isa_json, ena_mars_receipt) if DEBUG: diff --git a/mars-cli/tests/test_isa_json.py b/mars-cli/tests/test_isa_json.py index 25270f4..c4fd0d8 100644 --- a/mars-cli/tests/test_isa_json.py +++ b/mars-cli/tests/test_isa_json.py @@ -189,22 +189,20 @@ def test_update_study_materials_no_accession_categories(): updated_isa_json = update_isa_json(validated_isa_json, repo_response) # Check the accession number of the source - # Accession characteristic is of type String assert ( updated_isa_json.investigation.studies[0] .materials.sources[0] .characteristics[-1] - .value + .value.annotationValue == repo_response.accessions[0].value ) # Check the accession number of the sample - # Accession characteristic is of type String assert ( updated_isa_json.investigation.studies[0] .materials.samples[0] .characteristics[-1] - .value + .value.annotationValue == repo_response.accessions[1].value ) @@ -222,7 +220,6 @@ def test_update_study_materials_with_accession_categories(): updated_isa_json = update_isa_json(validated_isa_json, repo_response) # Check the accession number of the source - # Accession characteristic is of type OntologyAnnotation assert ( updated_isa_json.investigation.studies[0] .materials.sources[0] @@ -232,12 +229,11 @@ def test_update_study_materials_with_accession_categories(): ) # Check the accession number of the sample - # Accession characteristic is of type String assert ( updated_isa_json.investigation.studies[0] .materials.samples[0] .characteristics[-1] - .value + .value.annotationValue == repo_response.accessions[1].value ) From 6a91ee0326ba5f8f8026655e98a5f598d271a0a2 Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Wed, 6 Nov 2024 16:02:08 +0100 Subject: [PATCH 12/14] Keep Spotless happy --- .../controller/WebinIsaToXmlSubmissionController.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java b/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java index 9088199..e195d10 100644 --- a/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java +++ b/repository-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/controller/WebinIsaToXmlSubmissionController.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import org.dom4j.Document; import org.dom4j.DocumentHelper; @@ -148,8 +147,11 @@ public Map getBiosamples(List studies) { } private String getCharacteresticAnnotation(List characteristics) { - List filteredCharacteristics = characteristics.stream() - .filter(characteristic -> characteristic.category.id.contains("#characteristic_category/accession")) + List filteredCharacteristics = + characteristics.stream() + .filter( + characteristic -> + characteristic.category.id.contains("#characteristic_category/accession")) .collect(Collectors.toList()); if (filteredCharacteristics.isEmpty()) { From becc5b2e8595b10c558a45dbc28722e21c85851a Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Thu, 7 Nov 2024 09:39:16 +0100 Subject: [PATCH 13/14] Remove duplicated code --- mars-cli/mars_lib/submit.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mars-cli/mars_lib/submit.py b/mars-cli/mars_lib/submit.py index c31ee7e..8d79dee 100644 --- a/mars-cli/mars_lib/submit.py +++ b/mars-cli/mars_lib/submit.py @@ -88,17 +88,6 @@ def submission( ): raise ValueError("No target repository selected.") - if ( - TargetRepository.ENA in target_repositories - and data_file_paths - and file_transfer - ): - upload_to_ena( - file_paths=data_file_paths, - user_credentials=user_credentials, - submission_url=urls["ENA"]["DATA-SUBMISSION"], - file_transfer=file_transfer, - ) if TargetRepository.BIOSAMPLES in target_repositories: # Submit to Biosamples biosamples_result = submit_to_biosamples( From ba58c6c0ae3e02c9abc4d237bc20931697b0b63c Mon Sep 17 00:00:00 2001 From: Kevin De Pelseneer Date: Thu, 7 Nov 2024 10:07:37 +0100 Subject: [PATCH 14/14] Add output option --- mars-cli/mars_cli.py | 7 +++++++ mars-cli/mars_lib/submit.py | 10 ++++++---- mars-cli/tests/test_ftp_upload.py | 11 +++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/mars-cli/mars_cli.py b/mars-cli/mars_cli.py index 7b85a26..55a01e4 100644 --- a/mars-cli/mars_cli.py +++ b/mars-cli/mars_cli.py @@ -206,6 +206,11 @@ def cli(ctx, development): type=click.BOOL, help="Boolean indicating if the investigation is the root of the ISA JSON. Set this to True if the ISA-JSON does not contain a 'investigation' field.", ) +@click.option( + "--output", + type=click.STRING, + default=f"output_{datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}", +) @click.pass_context def submit( ctx, @@ -218,6 +223,7 @@ def submit( submit_to_metabolights, investigation_is_root, file_transfer, + output, data_files, ): """Start a submission to the target repositories.""" @@ -250,6 +256,7 @@ def submit( investigation_is_root, urls_dict, file_transfer, + output, data_file_paths, ) except requests.RequestException as err: diff --git a/mars-cli/mars_lib/submit.py b/mars-cli/mars_lib/submit.py index 8d79dee..dc347c9 100644 --- a/mars-cli/mars_lib/submit.py +++ b/mars-cli/mars_lib/submit.py @@ -30,7 +30,7 @@ def save_step_to_file(time_stamp: float, filename: str, isa_json: IsaJson): - dir_path = f"tmp/{str(time_stamp)}" + dir_path = f"tmp/{datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}" os.makedirs(dir_path, exist_ok=True) with open(f"{dir_path}/{filename}.json", "w") as f: @@ -49,8 +49,9 @@ def submission( investigation_is_root: bool, urls: dict[str, Any], file_transfer: str, + output: str, data_file_paths=None, -): +) -> None: # If credential manager info found: # Get password from the credential manager # Else: @@ -150,8 +151,9 @@ def submission( ) # TODO: Update `isa_json`, based on the receipt returned - # TODO: Return the updated ISA JSON - return isa_json + # Return the updated ISA JSON + with open(f"{output}.json", "w") as f: + f.write(isa_json.model_dump_json(by_alias=True, exclude_none=True)) def submit_to_biosamples( diff --git a/mars-cli/tests/test_ftp_upload.py b/mars-cli/tests/test_ftp_upload.py index 13915ab..a5d1672 100644 --- a/mars-cli/tests/test_ftp_upload.py +++ b/mars-cli/tests/test_ftp_upload.py @@ -13,10 +13,17 @@ def test_upload_login_failure(): uploader.upload([Path("./tests/fixtures/not_a_json_file.txt")]) -@pytest.mark.skip(reason="Relies on real ENA credentials in test_credentials_example.json") +@pytest.mark.skip( + reason="Relies on real ENA credentials in test_credentials_example.json" +) def test_upload_success(): # For local testing, add ENA username/password to test_credentials_example.json with open("./tests/test_credentials_example.json") as f: creds = json.load(f) uploader = FTPUploader("webin2.ebi.ac.uk", creds["username"], creds["password"]) - uploader.upload([Path("../test-data/ENA_TEST2.R1.fastq.gz"), Path("./tests/fixtures/not_a_json_file.txt")]) + uploader.upload( + [ + Path("../test-data/ENA_TEST2.R1.fastq.gz"), + Path("./tests/fixtures/not_a_json_file.txt"), + ] + )