From d28a5db21464d2f546c880007aa322314778b8e0 Mon Sep 17 00:00:00 2001 From: Shailesh Appukuttan Date: Thu, 19 Nov 2020 18:36:54 +0100 Subject: [PATCH 1/2] add edit return dict; updates to pytest --- hbp_validation_framework/__init__.py | 295 ++++++++++++------------- hbp_validation_framework/datastores.py | 20 +- hbp_validation_framework/utils.py | 83 +++---- setup.py | 2 +- test.txt | 1 - tests/conftest.py | 28 +-- tests/test_model_instances.py | 33 ++- tests/test_models.py | 56 +++-- tests/test_results.py | 12 +- tests/test_superuser.py | 63 +++--- tests/test_test_instances.py | 23 +- tests/test_tests.py | 110 +++++---- tests/test_utilities.py | 8 +- 13 files changed, 351 insertions(+), 383 deletions(-) delete mode 100644 test.txt diff --git a/hbp_validation_framework/__init__.py b/hbp_validation_framework/__init__.py index f187842..d9e14a0 100644 --- a/hbp_validation_framework/__init__.py +++ b/hbp_validation_framework/__init__.py @@ -53,6 +53,14 @@ def handle_response_error(message, response): full_message = "{}. Response = {}".format(message, response_text) raise ResponseError(full_message) +def renameNestedJSONKey(iterable, old_key, new_key): + if isinstance(iterable, list): + return [renameNestedJSONKey(item, old_key, new_key) for item in iterable] + if isinstance(iterable, dict): + for key in list(iterable.keys()): + if key == old_key: + iterable[new_key] = iterable.pop(key) + return iterable class HBPAuth(AuthBase): """Attaches OIDC Bearer Authentication to the given Request object.""" @@ -203,7 +211,7 @@ def _format_people_name(self, names): output_names_list.append({"given_name": "", "family_name": ""}) return output_names_list - # def exists_in_collab_else_create(self, project_id): + # def exists_in_collab_else_create(self, collab_id): # # TODO: needs to be updated for Collab v2 # """ # Checks with the hbp-collab-service if the Model Catalog / Validation Framework app @@ -211,10 +219,10 @@ def _format_people_name(self, names): # specified by the user (when run externally). # """ # try: - # url = "https://services.humanbrainproject.eu/collab/v0/collab/"+str(project_id)+"/nav/all/" + # url = "https://services.humanbrainproject.eu/collab/v0/collab/"+str(collab_id)+"/nav/all/" # response = requests.get(url, auth=HBPAuth(self.token), verify=self.verify) # except ValueError: - # print("Error contacting hbp-collab-service for Collab info. Possibly invalid Collab ID: {}".format(project_id)) + # print("Error contacting hbp-collab-service for Collab info. Possibly invalid Collab ID: {}".format(collab_id)) # for app_item in response.json(): # if app_item["app_id"] == str(self.app_id): @@ -222,7 +230,7 @@ def _format_people_name(self, names): # print ("Using existing {} app in this Collab. App nav ID: {}".format(self.app_name,app_nav_id)) # break # else: - # url = "https://services.humanbrainproject.eu/collab/v0/collab/"+str(project_id)+"/nav/root/" + # url = "https://services.humanbrainproject.eu/collab/v0/collab/"+str(collab_id)+"/nav/root/" # collab_root = requests.get(url, auth=HBPAuth(self.token), verify=self.verify).json()["id"] # import uuid # app_info = {"app_id": self.app_id, @@ -231,7 +239,7 @@ def _format_people_name(self, names): # "order_index": "-1", # "parent": collab_root, # "type": "IT"} - # url = "https://services.humanbrainproject.eu/collab/v0/collab/"+str(project_id)+"/nav/" + # url = "https://services.humanbrainproject.eu/collab/v0/collab/"+str(collab_id)+"/nav/" # headers = {'Content-type': 'application/json'} # response = requests.post(url, data=json.dumps(app_info), # auth=HBPAuth(self.token), headers=headers, @@ -250,7 +258,7 @@ def _format_people_name(self, names): # "app_type":"model_catalog", # "brain_region":"", # "cell_type":"", - # "project_id":8123, + # "collab_id":"model-validation", # "recording_modality":"", # "model_scope":"", # "abstraction_level":"", @@ -262,8 +270,8 @@ def _format_people_name(self, names): # "url":"https://validation-v1.brainsimulation.eu/parametersconfiguration-model-catalog/parametersconfigurationrest/" # } # """ - # if not config_data["config"]["project_id"]: - # raise ValueError("`project_id` cannot be empty!") + # if not config_data["config"]["collab_id"]: + # raise ValueError("`collab_id` cannot be empty!") # if not config_data["config"]["app_id"]: # raise ValueError("`app_id` cannot be empty!") # # check if the app has previously been configured: decide POST or PUT @@ -424,18 +432,14 @@ def __init__(self, username=None, password=None, environment="production", token self._set_app_info() def _set_app_info(self): - # TODO: check if needs to be updated for Collab v2 if self.environment == "production": - self.app_id = 360 self.app_name = "Validation Framework" elif self.environment == "dev": - self.app_id = 349 self.app_name = "Validation Framework (dev)" elif self.environment == "integration": - self.app_id = 432 self.app_name = "Model Validation app (staging)" - # def set_app_config(self, project_id="", only_if_new=False, recording_modality="", test_type="", species="", brain_region="", cell_type="", model_scope="", abstraction_level="", organization=""): + # def set_app_config(self, collab_id="", only_if_new=False, recording_modality="", test_type="", species="", brain_region="", cell_type="", model_scope="", abstraction_level="", organization=""): # # TODO: needs to be updated for Collab v2 # inputArgs = locals() # params = {} @@ -585,20 +589,17 @@ def list_tests(self, size=1000000, from_index=0, **filters): The filters may specify one or more attributes that belong to a test definition. The following test attributes can be specified: - * name * alias - * author - * species - * age + * name + * implementation_status * brain_region + * species * cell_type + * data_type * recording_modality * test_type * score_type - * model_scope - * abstraction_level - * data_type - * publication + * author Parameters ---------- @@ -621,8 +622,7 @@ def list_tests(self, size=1000000, from_index=0, **filters): >>> tests = test_library.list_tests(test_type="single cell activity", cell_type="Pyramidal Cell") """ - # TODO: verify valid filters for v2 APIs - valid_filters = ["name", "alias", "author", "species", "age", "brain_region", "cell_type", "recording_modality", "test_type", "score_type", "model_scope", "abstraction_level", "data_type", "publication"] + valid_filters = ["alias", "name", "implementation_status", "brain_region", "species", "cell_type", "data_type", "recording_modality", "test_type", "score_type", "author"] params = locals()["filters"] for filter in params: if filter not in valid_filters: @@ -679,8 +679,8 @@ def add_test(self, name=None, alias=None, author=None, Returns ------- - UUID - UUID of the test instance that has been created. + dict + data of test instance that has been created. Examples -------- @@ -704,10 +704,11 @@ def add_test(self, name=None, alias=None, author=None, values = self.get_attribute_options() for field in ("species", "brain_region", "cell_type", "recording_modality", "test_type", "score_type"): if field in test_data and test_data[field] not in values[field] + [None]: - raise Exception(field + " = '" + test_data[field] + "' is invalid.\nValue has to be one of these: " + values[field]) + raise Exception("{} = '{}' is invalid.\nValue has to be one of these: {}".format(field, test_data[field], values[field])) # format names of authors as required by API - test_data["author"] = self._format_people_name(test_data["author"]) + if "author" in test_data: + test_data["author"] = self._format_people_name(test_data["author"]) # 'data_location' is now a list of urls if not isinstance(test_data["data_location"], list): @@ -719,7 +720,7 @@ def add_test(self, name=None, alias=None, author=None, auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 201: - return response.json()["id"] + return response.json() else: handle_response_error("Error in adding test", response) @@ -772,8 +773,8 @@ def edit_test(self, test_id=None, name=None, alias=None, author=None, Returns ------- - UUID - (Verify!) UUID of the test instance that has been edited. + data + data of test instance that has been edited. Examples -------- @@ -782,7 +783,7 @@ def edit_test(self, test_id=None, name=None, alias=None, author=None, test_type="network structure", score_type="Other", protocol="To be filled sometime later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/hippounit/feat_CA1_pyr_cACpyr_more_features.json", data_type="Mean, SD") """ - if test_id == "": + if not test_id: raise Exception("Test ID needs to be provided for editing a test.") test_data = {} @@ -796,10 +797,11 @@ def edit_test(self, test_id=None, name=None, alias=None, author=None, values = self.get_attribute_options() for field in ("species", "brain_region", "cell_type", "recording_modality", "test_type", "score_type"): if field in test_data and test_data[field] not in values[field] + [None]: - raise Exception(field + " = '" + test_data[field] + "' is invalid.\nValue has to be one of these: " + values[field]) + raise Exception("{} = '{}' is invalid.\nValue has to be one of these: {}".format(field, test_data[field], values[field])) # format names of authors as required by API - test_data["author"] = self._format_people_name(test_data["author"]) + if "author" in test_data: + test_data["author"] = self._format_people_name(test_data["author"]) # 'data_location' is now a list of urls if not isinstance(test_data["data_location"], list): @@ -811,7 +813,7 @@ def edit_test(self, test_id=None, name=None, alias=None, author=None, auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 200: - return response.json()["id"] + return response.json() else: handle_response_error("Error in editing test", response) @@ -974,7 +976,7 @@ def add_test_instance(self, test_id="", alias="", repository="", path="", versio """Register a new test instance. This allows to add a new instance to an existing test in the test library. - The `test_id` needs to be specified as input parameter. + The `test_id` or `alias` needs to be specified as input parameter. Parameters ---------- @@ -995,46 +997,36 @@ def add_test_instance(self, test_id="", alias="", repository="", path="", versio Returns ------- - UUID - UUID of the test instance that has been created. - - Note - ---- - * `alias` is not currently implemented in the API; kept for future use. - * TODO: Either test_id or alias needs to be provided, with test_id taking precedence over alias. + dict + data of test instance that has been created. Examples -------- - >>> response = test_library.add_test_instance(test_id="7b63f87b-d709-4194-bae1-15329daf3dec", + >>> instance = test_library.add_test_instance(test_id="7b63f87b-d709-4194-bae1-15329daf3dec", repository="https://github.com/appukuttan-shailesh/morphounit.git", path="morphounit.tests.CellDensityTest", version="3.0") """ - instance_data = {} - if repository: - instance_data["repository"] = repository - if path: - instance_data["path"] = path - if version: - instance_data["version"] = version - if description: - instance_data["description"] = description - if parameters: - instance_data["parameters"] = parameters + instance_data = locals() + instance_data.pop("self") + + for key, val in instance_data.items(): + if val == "": + instance_data[key] = None test_id = test_id or alias if not test_id: raise Exception("test_id or alias needs to be provided for finding the test.") else: - url = self.url + "/tests/" + test_id + "/instances/" + url = self.url + "/tests/" + quote(test_id) + "/instances/" headers = {'Content-type': 'application/json'} response = requests.post(url, data=json.dumps(instance_data), auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 201: - return response.json()["id"] + return response.json() else: handle_response_error("Error in adding test instance", response) @@ -1074,12 +1066,12 @@ def edit_test_instance(self, instance_id="", test_id="", alias="", repository=No Returns ------- - UUID - UUID of the test instance that has was edited. + dict + data of test instance that has was edited. Examples -------- - >>> response = test_library.edit_test_instance(test_id="7b63f87b-d709-4194-bae1-15329daf3dec", + >>> instance = test_library.edit_test_instance(test_id="7b63f87b-d709-4194-bae1-15329daf3dec", repository="https://github.com/appukuttan-shailesh/morphounit.git", path="morphounit.tests.CellDensityTest", version="4.0") @@ -1109,7 +1101,7 @@ def edit_test_instance(self, instance_id="", test_id="", alias="", repository=No response = requests.put(url, data=json.dumps(instance_data), auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 200: - return response.json()["id"] + return response.json() else: handle_response_error("Error in editing test instance", response) @@ -1240,7 +1232,7 @@ def get_result(self, result_id=""): response = requests.get(url, auth=self.auth, verify=self.verify) if response.status_code != 200: handle_response_error("Error in retrieving result", response) - result_json = response.json() + result_json = renameNestedJSONKey(response.json(), "project_id", "collab_id") return result_json def list_results(self, size=1000000, from_index=0, **filters): @@ -1277,9 +1269,9 @@ def list_results(self, size=1000000, from_index=0, **filters): if response.status_code != 200: handle_response_error("Error in retrieving results", response) result_json = response.json() - return result_json + return renameNestedJSONKey(result_json, "project_id", "collab_id") - def register_result(self, test_result, data_store=None, project_id=None): + def register_result(self, test_result, data_store=None, collab_id=None): """Register test result with HBP Validation Results Service. The score of a test, along with related output data such as figures, @@ -1291,7 +1283,7 @@ def register_result(self, test_result, data_store=None, project_id=None): a :class:`sciunit.Score` instance returned by `test.judge(model)` data_store : :class:`DataStore` a :class:`DataStore` instance, for uploading related data generated by the test run, e.g. figures. - project_id : str + collab_id : str String input specifying the Collab path, e.g. 'model-validation' to indicate Collab 'https://wiki.ebrains.eu/bin/view/Collabs/model-validation/'. This is used to indicate the Collab where results should be saved. @@ -1302,8 +1294,8 @@ def register_result(self, test_result, data_store=None, project_id=None): Returns ------- - UUID - UUID of the test result that has been created. + dict + data of test result that has been created. Examples -------- @@ -1311,13 +1303,13 @@ def register_result(self, test_result, data_store=None, project_id=None): >>> response = test_library.register_result(test_result=score) """ - if project_id is None: - project_id = test_result.related_data.get("project_id", None) - if project_id is None: - raise Exception("Don't know where to register this result. Please specify `project_id`!") + if collab_id is None: + collab_id = test_result.related_data.get("collab_id", None) + if collab_id is None: + raise Exception("Don't know where to register this result. Please specify `collab_id`!") model_catalog = ModelCatalog.from_existing(self) - model_instance_uuid = model_catalog.find_model_instance_else_add(test_result.model) + model_instance_uuid = model_catalog.find_model_instance_else_add(test_result.model)["id"] results_storage = [] if data_store: @@ -1325,13 +1317,14 @@ def register_result(self, test_result, data_store=None, project_id=None): data_store.authorize(self.auth) # relies on data store using HBP authorization # if this is not the case, need to authenticate/authorize # the data store before passing to `register()` - if data_store.project_id is None: - data_store.project_id = project_id + if data_store.collab_id is None: + data_store.collab_id = collab_id files_to_upload = [] if "figures" in test_result.related_data: files_to_upload.extend(test_result.related_data["figures"]) if files_to_upload: - results_storage.append(data_store.upload_data(files_to_upload)) + list_dict_files_to_upload = [{"download_url": f["filepath"], "size": f["filesize"]} for f in data_store.upload_data(files_to_upload)] + results_storage.extend(list_dict_files_to_upload) url = self.url + "/results/" result_json = { @@ -1341,7 +1334,7 @@ def register_result(self, test_result, data_store=None, project_id=None): "score": int(test_result.score) if isinstance(test_result.score, bool) else test_result.score, "passed": None if "passed" not in test_result.related_data else test_result.related_data["passed"], #"platform": str(self._get_platform()), # not currently supported in v2 - "project_id": project_id, + "project_id": collab_id, "normalized_score": int(test_result.score) if isinstance(test_result.score, bool) else test_result.score, } @@ -1351,7 +1344,7 @@ def register_result(self, test_result, data_store=None, project_id=None): verify=self.verify) if response.status_code == 201: print("Result registered successfully!") - return response.json()["id"] + return renameNestedJSONKey(response.json(), "project_id", "collab_id") else: handle_response_error("Error registering result", response) @@ -1474,18 +1467,14 @@ def __init__(self, username=None, password=None, environment="production", token self._set_app_info() def _set_app_info(self): - # TODO: check if needs to be updated for Collab v2 if self.environment == "production": - self.app_id = 357 self.app_name = "Model Catalog" elif self.environment == "dev": - self.app_id = 348 self.app_name = "Model Catalog (dev)" elif self.environment == "integration": - self.app_id = 431 self.app_name = "Model Catalog (staging)" - # def set_app_config(self, project_id="", only_if_new=False, species="", brain_region="", cell_type="", model_scope="", abstraction_level="", organization=""): + # def set_app_config(self, collab_id="", only_if_new=False, species="", brain_region="", cell_type="", model_scope="", abstraction_level="", organization=""): # # TODO: needs to be updated for Collab v2 # inputArgs = locals() # params = {} @@ -1526,7 +1515,7 @@ def _set_app_info(self): # organization.append(model["organization"]) # filters = {} - # for key in ["project_id", "app_id", "species", "brain_region", "cell_type", "model_scope", "abstraction_level", "organization"]: + # for key in ["collab_id", "app_id", "species", "brain_region", "cell_type", "model_scope", "abstraction_level", "organization"]: # if isinstance(locals()[key], list): # filters[key] = ",".join(locals()[key]) # else: @@ -1586,7 +1575,7 @@ def get_model(self, model_id="", alias="", instances=True, images=True): model_json.pop("instances") if images is False: model_json.pop("images") - return model_json + return renameNestedJSONKey(model_json, "project_id", "collab_id") def list_models(self, size=1000000, from_index=0, **filters): """Retrieve list of model descriptions satisfying specified filters. @@ -1594,19 +1583,18 @@ def list_models(self, size=1000000, from_index=0, **filters): The filters may specify one or more attributes that belong to a model description. The following model attributes can be specified: - * project_id - * name * alias - * author - * organization - * species + * name * brain_region + * species * cell_type * model_scope * abstraction_level + * author * owner - * project - * license + * organization + * collab_id + * private Parameters ---------- @@ -1625,16 +1613,19 @@ def list_models(self, size=1000000, from_index=0, **filters): Examples -------- >>> models = model_catalog.list_models() - >>> models = model_catalog.list_models(project_id="39968") + >>> models = model_catalog.list_models(collab_id="model-validation") >>> models = model_catalog.list_models(cell_type="Pyramidal Cell", brain_region="Hippocampus") """ - # TODO: verify valid filters for v2 APIs - valid_filters = ["project_id", "name", "alias", "author", "organization", "species", "brain_region", "cell_type", "model_scope", "abstraction_level", "owner", "project", "license"] + valid_filters = ["name", "alias", "brain_region", "species", "cell_type", "model_scope", "abstraction_level", "author", "owner", "organization", "collab_id", "private"] params = locals()["filters"] for filter in params: if filter not in valid_filters: raise ValueError("The specified filter '{}' is an invalid filter!\nValid filters are: {}".format(filter, valid_filters)) + + # handle naming difference with API: collab_id <-> project_id + if "collab_id" in params: + params["project_id"] = params.pop("collab_id") url = self.url + "/models/" url += "?" + urlencode(params) + "&size=" + str(size) + "&from_index=" + str(from_index) @@ -1643,9 +1634,9 @@ def list_models(self, size=1000000, from_index=0, **filters): models = response.json() except (json.JSONDecodeError, simplejson.JSONDecodeError): handle_response_error("Error in list_models()", response) - return models + return renameNestedJSONKey(models, "project_id", "collab_id") - def register_model(self, project_id=None, name=None, alias=None, author=None, owner=None, organization=None, private=False, + def register_model(self, collab_id=None, name=None, alias=None, author=None, owner=None, organization=None, private=False, species=None, brain_region=None, cell_type=None, model_scope=None, abstraction_level=None, project=None, license=None, description=None, instances=[], images=[]): """Register a new model in the model catalog. @@ -1656,7 +1647,7 @@ def register_model(self, project_id=None, name=None, alias=None, author=None, ow Parameters ---------- - project_id : string + collab_id : string Specifies the ID of the host collab in the HBP Collaboratory. (the model would belong to this collab) name : string @@ -1694,14 +1685,14 @@ def register_model(self, project_id=None, name=None, alias=None, author=None, ow Returns ------- - UUID - UUID of the model description that has been created. + dict + Model description that has been created. Examples -------- (without instances and images) - >>> model = model_catalog.register_model(project_id="39968", name="Test Model - B2", + >>> model = model_catalog.register_model(collab_id="model-validation", name="Test Model - B2", alias="Model vB2", author="Shailesh Appukuttan", organization="HBP-SP6", private=False, cell_type="Granule Cell", model_scope="Single cell model", abstraction_level="Spiking neurons", @@ -1711,7 +1702,7 @@ def register_model(self, project_id=None, name=None, alias=None, author=None, ow (with instances and images) - >>> model = model_catalog.register_model(project_id="39968", name="Test Model - C2", + >>> model = model_catalog.register_model(collab_id="model-validation", name="Test Model - C2", alias="Model vC2", author="Shailesh Appukuttan", organization="HBP-SP6", private=False, cell_type="Granule Cell", model_scope="Single cell model", abstraction_level="Spiking neurons", @@ -1730,17 +1721,21 @@ def register_model(self, project_id=None, name=None, alias=None, author=None, ow model_data = {} args = locals() + + # handle naming difference with API: collab_id <-> project_id + args["project_id"] = args.pop("collab_id") + for field in ["project_id", "name", "alias", "author", "organization", "private", "cell_type", "model_scope", "abstraction_level", "brain_region", "species", "owner", "project", "license", "description", "instances", "images"]: - if args[field]: + if args[field] or field == "private": model_data[field] = args[field] values = self.get_attribute_options() for field in ["species", "brain_region", "cell_type", "abstraction_level", "model_scope"]: if field in model_data and model_data[field] not in values[field] + [None]: - raise Exception(field + " = '" + model_data[field] + "' is invalid.\nValue has to be one of these: " + str(values[field])) + raise Exception("{} = '{}' is invalid.\nValue has to be one of these: {}".format(field, model_data[field], values[field])) if private not in [True, False]: raise Exception("Model's 'private' attribute should be specified as True / False. Default value is False.") @@ -1756,11 +1751,11 @@ def register_model(self, project_id=None, name=None, alias=None, author=None, ow auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 201: - return response.json()["id"] + return renameNestedJSONKey(response.json(), "project_id", "collab_id") else: handle_response_error("Error in adding model", response) - def edit_model(self, model_id=None, project_id=None, name=None, alias=None, author=None, owner=None, organization=None, private=None, + def edit_model(self, model_id=None, collab_id=None, name=None, alias=None, author=None, owner=None, organization=None, private=None, species=None, brain_region=None, cell_type=None, model_scope=None, abstraction_level=None, project=None, license=None, description=None): """Edit an existing model on the model catalog. @@ -1773,7 +1768,7 @@ def edit_model(self, model_id=None, project_id=None, name=None, alias=None, auth ---------- model_id : UUID System generated unique identifier associated with model description. - project_id : string + collab_id : string Specifies the ID of the host collab in the HBP Collaboratory. (the model would belong to this collab) name : string @@ -1812,12 +1807,12 @@ def edit_model(self, model_id=None, project_id=None, name=None, alias=None, auth Returns ------- - UUID - UUID of the model description that has been edited. + dict + Model description that has been edited. Examples -------- - >>> model = model_catalog.edit_model(project_id="39968", name="Test Model - B2", + >>> model = model_catalog.edit_model(collab_id="model-validation", name="Test Model - B2", model_id="8c7cb9f6-e380-452c-9e98-e77254b088c5", alias="Model-B2", author="Shailesh Appukuttan", organization="HBP-SP6", private=False, cell_type="Granule Cell", model_scope="Single cell model", @@ -1827,31 +1822,35 @@ def edit_model(self, model_id=None, project_id=None, name=None, alias=None, auth description="This is a test entry") """ - if model_id == "": + if not model_id: raise Exception("Model ID needs to be provided for editing a model.") model_data = {} args = locals() + + # handle naming difference with API: collab_id <-> project_id + args["project_id"] = args.pop("collab_id") + for field in ["project_id", "name", "alias", "author", "organization", "private", "cell_type", "model_scope", "abstraction_level", "brain_region", "species", "owner", "project", "license", "description"]: - if args[field]: + if args[field] or (field == "private" and args[field] != None): model_data[field] = args[field] values = self.get_attribute_options() for field in ("species", "brain_region", "cell_type", "abstraction_level", "model_scope"): if field in model_data and model_data[field] not in values[field] + [None]: - raise Exception(key + " = '" + model_data[key] + "' is invalid.\nValue has to be one of these: " + str(values[key])) + raise Exception("{} = '{}' is invalid.\nValue has to be one of these: {}".format(field, model_data[field], values[field])) if private and private not in [True, False]: raise Exception("Model's 'private' attribute should be specified as True / False. Default value is False.") # format names of authors and owners as required by API for field in ("author", "owner"): - if model_data[field]: + if model_data.get(field): model_data[field] = self._format_people_name(model_data[field]) - if model_data["alias"] == "": + if "alias" in model_data and model_data["alias"] == "": model_data["alias"] = None if model_data.get("private", False) not in (True, False): @@ -1863,7 +1862,7 @@ def edit_model(self, model_id=None, project_id=None, name=None, alias=None, auth auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 200: - return response.json()["id"] + return renameNestedJSONKey(response.json(), "project_id", "collab_id") else: handle_response_error("Error in updating model", response) @@ -2049,7 +2048,7 @@ def download_model_instance(self, instance_path="", instance_id="", model_id="", # ***** Handles Collab storage urls ***** repo_id = model_source.split("drive.ebrains.eu/lib/")[1].split("/")[0] model_path = "/" + "/".join(model_source.split("drive.ebrains.eu/lib/")[1].split("/")[2:]) - datastore = URI_SCHEME_MAP["collab_v2"](project_id=repo_id, auth=self.auth) + datastore = URI_SCHEME_MAP["collab_v2"](collab_id=repo_id, auth=self.auth) fileList = datastore.download_data(model_path, local_directory=local_directory, overwrite=overwrite) elif model_source.startswith("swift://cscs.ch/"): # ***** Handles CSCS private urls ***** @@ -2141,7 +2140,7 @@ def add_model_instance(self, model_id="", alias="", source="", version="", descr """Register a new model instance. This allows to add a new instance of an existing model in the model catalog. - The `model_id` needs to be specified as input parameter. + The `model_id` or 'alias' needs to be specified as input parameter. Parameters ---------- @@ -2168,17 +2167,12 @@ def add_model_instance(self, model_id="", alias="", source="", version="", descr Returns ------- - UUID - UUID of the model instance that has been created. - - Note - ---- - * `alias` is not currently implemented in the API; kept for future use. - * TODO: Either model_id or alias needs to be provided, with model_id taking precedence over alias. + dict + data of model instance that has been created. Examples -------- - >>> instance_id = model_catalog.add_model_instance(model_id="196b89a3-e672-4b96-8739-748ba3850254", + >>> instance = model_catalog.add_model_instance(model_id="196b89a3-e672-4b96-8739-748ba3850254", source="https://www.abcde.com", version="1.0", description="basic model variant", @@ -2196,19 +2190,18 @@ def add_model_instance(self, model_id="", alias="", source="", version="", descr if val == "": instance_data[key] = None - if model_id == "" and alias == "": - raise Exception("Model ID needs to be provided for finding the model.") - #raise Exception("Model ID or alias needs to be provided for finding the model.") - elif model_id != "": - url = self.url + "/models/" + model_id + "/instances/" + model_id = model_id or alias + if not model_id: + raise Exception("model_id or alias needs to be provided for finding the model.") else: - url = self.url + "/models/" + quote(alias) + "/instances/" + url = self.url + "/models/" + quote(model_id) + "/instances/" + headers = {'Content-type': 'application/json'} response = requests.post(url, data=json.dumps(instance_data), auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 201: - return response.json()["id"] + return response.json() else: handle_response_error("Error in adding model instance", response) @@ -2225,38 +2218,40 @@ def find_model_instance_else_add(self, model_obj): Returns ------- - UUID - UUID of the existing or created model instance. + dict + data of existing or created model instance. Note ---- * `model_obj` is expected to contain the attribute `model_instance_uuid`, - or both the attributes `model_uuid` and `model_version`. + or both the attributes `model_uuid`/`model_alias` and `model_version`. Examples -------- - >>> instance_id = model_catalog.find_model_instance_else_add(model) + >>> instance = model_catalog.find_model_instance_else_add(model) """ if not getattr(model_obj, "model_instance_uuid", None): # check that the model is registered with the model registry. - if not hasattr(model_obj, "model_uuid"): - raise AttributeError("Model object does not have a 'model_uuid' attribute. " - "Please register it with the Validation Framework and add the 'model_uuid' to the model object.") + if not hasattr(model_obj, "model_uuid") and not hasattr(model_obj, "model_alias"): + raise AttributeError("Model object does not have a 'model_uuid'/'model_alias' attribute. " + "Please register it with the Validation Framework and add the 'model_uuid'/'model_alias' to the model object.") if not hasattr(model_obj, "model_version"): raise AttributeError("Model object does not have a 'model_version' attribute") - try: - model_instance_uuid = self.get_model_instance(model_id=model_obj.model_uuid, - version=model_obj.model_version)['id'] - except Exception: # probably the instance doesn't exist (todo: distinguish from other reasons for Exception) - # so we create a new instance - model_instance_uuid = self.add_model_instance(model_id=model_obj.model_uuid, + + model_instance = self.get_model_instance(model_id=getattr(model_obj, "model_uuid", None), + alias=getattr(model_obj, "model_alias", None), + version=model_obj.model_version) + if not model_instance: # check if instance doesn't exist + # if yes, then create a new instance + model_instance = self.add_model_instance(model_id=getattr(model_obj, "model_uuid", None), + alias=getattr(model_obj, "model_alias", None), source=getattr(model_obj, "remote_url", ""), version=model_obj.model_version, parameters=getattr(model_obj, "parameters", "")) else: - model_instance_uuid = model_obj.model_instance_uuid - return model_instance_uuid + model_instance = self.get_model_instance(instance_id=model_obj.model_instance_uuid) + return model_instance def edit_model_instance(self, instance_id="", model_id="", alias="", source=None, version=None, description=None, parameters=None, code_format=None, hash=None, morphology=None, license=None): """Edit an existing model instance. @@ -2300,12 +2295,12 @@ def edit_model_instance(self, instance_id="", model_id="", alias="", source=None Returns ------- - UUID - UUID of the model instance that has been edited. + dict + data of model instance that has been edited. Examples -------- - >>> instance_id = model_catalog.edit_model_instance(instance_id="fd1ab546-80f7-4912-9434-3c62af87bc77", + >>> instance = model_catalog.edit_model_instance(instance_id="fd1ab546-80f7-4912-9434-3c62af87bc77", source="https://www.abcde.com", version="1.0", description="passive model variant", @@ -2341,7 +2336,7 @@ def edit_model_instance(self, instance_id="", model_id="", alias="", source=None response = requests.put(url, data=json.dumps(instance_data), auth=self.auth, headers=headers, verify=self.verify) if response.status_code == 200: - return response.json()["id"] + return response.json() else: handle_response_error("Error in editing model instance at {}".format(url), response) diff --git a/hbp_validation_framework/datastores.py b/hbp_validation_framework/datastores.py index 5b3eb31..7fa7a5a 100644 --- a/hbp_validation_framework/datastores.py +++ b/hbp_validation_framework/datastores.py @@ -26,7 +26,7 @@ import requests -import hbp_seafile +import ebrains_drive mimetypes.init() @@ -49,8 +49,8 @@ class CollabV2DataStore(object): A class for uploading and downloading data from HBP Collaboratory v2 seafile storage. """ - def __init__(self, project_id=None, base_folder="/", auth=None, **kwargs): - self.project_id = project_id + def __init__(self, collab_id=None, base_folder="/", auth=None, **kwargs): + self.collab_id = collab_id self.base_folder = base_folder.strip("/") self._auth = auth # we defer authorization until needed self._authorized = False @@ -62,9 +62,9 @@ def authorized(self): def authorize(self, auth=None): if auth is None: auth = self._auth - self.client = hbp_seafile.connect(token=auth.token) + self.client = ebrains_drive.connect(token=auth.token) self._authorized = True - self.repo = self.client.repos.get_repo_by_url(self.project_id) + self.repo = self.client.repos.get_repo_by_url(self.collab_id) def upload_data(self, file_paths, overwrite=False): if not self.authorized: @@ -93,7 +93,8 @@ def upload_data(self, file_paths, overwrite=False): filename = os.path.basename(relative_path) seafdir = self.repo.get_dir(parent) file_entity = seafdir.upload_local_file(local_path, overwrite=overwrite) - uploaded_file_paths.append(upload_path_prefix + file_entity.path) + uploaded_file_paths.append({"filepath": upload_path_prefix + file_entity.path, "filesize": file_entity.size}) + # uploaded_file_paths.append(file_entity._get_download_link()) # this does not work as the link changes with token return uploaded_file_paths def _make_folders(self, folder_path, parent): @@ -205,7 +206,7 @@ def load_data(self, remote_path): class SwiftDataStore(object): """ - A class for downloading data from CSCS Swift storage. + A class for uploading and downloading data from CSCS Swift storage. Note: data from public containers can also be downloaded via `HTTPDataStore` """ @@ -228,7 +229,10 @@ def upload_data(self, file_paths, username="", container="", project=None, remot container = input("Please enter target container name: ") container_obj = Container(container, username, project=project) remote_paths = container_obj.upload(file_paths, remote_directory=remote_directory, overwrite=overwrite) - return remote_paths + uploaded_file_paths = [] + for ind, f in enumerate(file_paths): + uploaded_file_paths.append({"filepath": remote_paths[ind], "filesize": os.path.getsize(f)}) + return uploaded_file_paths def get_container(self, remote_path, username=""): try: diff --git a/hbp_validation_framework/utils.py b/hbp_validation_framework/utils.py index 6e3a8ba..b10ab08 100644 --- a/hbp_validation_framework/utils.py +++ b/hbp_validation_framework/utils.py @@ -278,7 +278,7 @@ def run_test_offline(model="", test_config_file=""): pickle.dump(score, file) return test_result_file -def upload_test_result(username="", password=None, environment="production", test_result_file="", storage_project_id="", register_result=True, client_obj=None): +def upload_test_result(username="", password=None, environment="production", test_result_file="", storage_collab_id="", register_result=True, client_obj=None): """Register the result with the Validation Service This method will register the validation result specified via the test result file @@ -298,7 +298,7 @@ def upload_test_result(username="", password=None, environment="production", tes be read (the latter is currently not implemented). test_result_file : string Absolute path of the test result file generated by :meth:`run_test_offline` - storage_project_id : string + storage_collab_id : string Collab ID where output files should be stored; if empty, stored in model's host Collab. register_result : boolean Specify whether the test results are to be scored on the validation framework. @@ -315,8 +315,8 @@ def upload_test_result(username="", password=None, environment="production", tes Returns ------- - UUID - UUID of the test result that has been created. + dict + data of test result that has been created. object score object evaluated by the test. @@ -340,15 +340,15 @@ def upload_test_result(username="", password=None, environment="production", tes model_catalog = ModelCatalog.from_existing(client_obj) else: model_catalog = ModelCatalog(username, password, environment=environment) - model_instance_uuid = model_catalog.find_model_instance_else_add(score.model) + model_instance_uuid = model_catalog.find_model_instance_else_add(score.model)["id"] model_instance_json = model_catalog.get_model_instance(instance_id=model_instance_uuid) model_json = model_catalog.get_model(model_id=model_instance_json["model_id"]) - model_host_project_id = model_json["project_id"] + model_host_collab_id = model_json["collab_id"] model_name = model_json["name"] - if not storage_project_id: - storage_project_id = model_host_project_id - score.related_data["project_id"] = storage_project_id + if not storage_collab_id: + storage_collab_id = model_host_collab_id + score.related_data["collab_id"] = storage_collab_id # Check if result with same hash has already been uploaded for # this (model instance, test instance) combination; if yes, don't register result @@ -369,14 +369,14 @@ def upload_test_result(username="", password=None, environment="production", tes # `.replace(" ", "_")` used to avoid Collab storage path errors due to spaces collab_folder = "validation_results/{}/{}_{}".format(datetime.now().strftime("%Y-%m-%d"),model_name.replace(" ", "_"), datetime.now().strftime("%Y%m%d-%H%M%S")) - collab_storage = CollabV2DataStore(project_id=storage_project_id, + collab_storage = CollabV2DataStore(collab_id=storage_collab_id, base_folder=collab_folder, auth=test_library.auth) response = test_library.register_result(test_result=score, data_store=collab_storage) return response, score -def run_test(username="", password=None, environment="production", model="", test_instance_id="", test_id="", test_alias="", test_version="", storage_project_id="", register_result=True, client_obj=None, **params): +def run_test(username="", password=None, environment="production", model="", test_instance_id="", test_id="", test_alias="", test_version="", storage_collab_id="", register_result=True, client_obj=None, **params): """Run validation test and register result This will execute the following methods by relaying the output of one to the next: @@ -406,7 +406,7 @@ def run_test(username="", password=None, environment="production", model="", tes User-assigned unique identifier associated with test definition. test_version : string User-assigned identifier (unique for each test) associated with test instance. - storage_project_id : string + storage_collab_id : string Collab ID where output files should be stored; if empty, stored in model's host Collab. register_result : boolean Specify whether the test results are to be scored on the validation framework. @@ -425,24 +425,24 @@ def run_test(username="", password=None, environment="production", model="", tes Returns ------- - UUID - UUID of the test result that has been created. + dict + data of test result that has been created. object score object evaluated by the test. Examples -------- - >>> result_id, score = utils.run_test(username="HBP_USERNAME", password="HBP_PASSWORD" environment="production", model=cell_model, test_alias="basalg_msn_d1", test_version="1.0", storage_project_id="8123", register_result=True) + >>> result_id, score = utils.run_test(username="HBP_USERNAME", password="HBP_PASSWORD" environment="production", model=cell_model, test_alias="basalg_msn_d1", test_version="1.0", storage_collab_id="8123", register_result=True) """ test_config_file = prepare_run_test_offline(username=username, password=password, environment=environment, test_instance_id=test_instance_id, test_id=test_id, test_alias=test_alias, test_version=test_version, client_obj=client_obj, **params) test_result_file = run_test_offline(model=model, test_config_file=test_config_file) - result_id, score = upload_test_result(username=username, password=password, environment=environment, test_result_file=test_result_file, storage_project_id=storage_project_id, register_result=register_result, client_obj=client_obj) + result_id, score = upload_test_result(username=username, password=password, environment=environment, test_result_file=test_result_file, storage_collab_id=storage_collab_id, register_result=register_result, client_obj=client_obj) return result_id, score def generate_HTML_report(username="", password=None, environment="production", model_list=[], model_instance_list=[], test_list=[], test_instance_list=[], - result_list=[], project_id=None, client_obj=None): + result_list=[], show_links=True, client_obj=None): """Generates an HTML report for specified test results This method will generate an HTML report for the specified test results. @@ -467,9 +467,9 @@ def generate_HTML_report(username="", password=None, environment="production", m List of test instance UUIDs for which score matrix is to be generated. result_list : list List of result UUIDs for which score matrix is to be generated. - project_id : string, optional - Collaboratory ID where hyperlinks to results are to be redirected. - If unspecified, these data units will not have clickable hyperlinks. + show_links : boolean, optional + To specify if hyperlinks to results are to be provided. + If false, these data units will not have clickable hyperlinks. client_obj : ModelCatalog/TestLibrary object Used to easily create a new ModelCatalog/TestLibrary object if either exist already. Avoids need for repeated authentications; improves performance. Also, helps minimize @@ -502,13 +502,6 @@ def generate_HTML_report(username="", password=None, environment="production", m model_catalog = ModelCatalog(username, password, environment=environment) test_library = TestLibrary.from_existing(model_catalog) - if project_id: - # check if app exists; if not then create - MCapp_navID = model_catalog.exists_in_collab_else_create(project_id) - model_catalog.set_app_config(project_id=project_id, app_id=MCapp_navID, only_if_new="True") - VFapp_navID = test_library.exists_in_collab_else_create(project_id) - test_library.set_app_config(project_id=project_id, app_id=VFapp_navID, only_if_new="True") - # retrieve all model instances from specified models if model_list: for entry in model_list: @@ -543,8 +536,6 @@ def generate_HTML_report(username="", password=None, environment="production", m # remove duplicate result UUIDs result_list = list(collections.OrderedDict.fromkeys(result_list).keys()) - # TODO: ensure atleast one valid result UUID to be evaluated; else create no report, alert user - # utilize each result entry result_summary_table = [] # list of dicts, each with 4 keys -> result_id, model_label, test_label, score list_results = [] @@ -569,11 +560,10 @@ def generate_HTML_report(username="", password=None, environment="production", m model_label = (model["alias"] if model["alias"] else model["name"]) + " (" + str(model_instance["version"]) + ")" test_label = (test["alias"] if test["alias"] else test["name"]) + " (" + str(test_instance["version"]) + ")" - if project_id: - # TODO: update URLs for Collab v2 - result_url = "https://collab.humanbrainproject.eu/#/collab/{}/nav/{}?state=result.{}".format(str(project_id),str(VFapp_navID), r_id) - model_url = "https://collab.humanbrainproject.eu/#/collab/{}/nav/{}?state=model.{}".format(str(project_id),str(MCapp_navID), model["id"]) - test_url = "https://collab.humanbrainproject.eu/#/collab/{}/nav/{}?state=test.{}".format(str(project_id),str(VFapp_navID), test["id"]) + if show_links: + result_url = "https://model-catalog.brainsimulation.eu/#result_id.{}".format(r_id) + model_url = "https://model-catalog.brainsimulation.eu/#model_id.{}".format(model["id"]) + test_url = "https://model-catalog.brainsimulation.eu/#test_id.{}".format(test["id"]) result_summary_table.append({"result_id": (r_id, result_url), "model_label": (model_label, model_url), "test_label": (test_label, test_url), @@ -608,7 +598,7 @@ def generate_HTML_report(username="", password=None, environment="production", m def generate_PDF_report(html_report_path=None, username="", password=None, environment="production", model_list=[], model_instance_list=[], - test_list=[], test_instance_list=[], result_list=[], project_id=None, + test_list=[], test_instance_list=[], result_list=[], show_links=True, only_results=False, client_obj=None): """Generates a PDF report for specified test results @@ -639,9 +629,9 @@ def generate_PDF_report(html_report_path=None, username="", password=None, List of test instance UUIDs for which score matrix is to be generated. result_list : list List of result UUIDs for which score matrix is to be generated. - project_id : string, optional - Collaboratory ID where hyperlinks to results are to be redirected. - If unspecified, these data units will not have clickable hyperlinks. + show_links : boolean, optional + To specify if hyperlinks to results are to be provided. + If false, these data units will not have clickable hyperlinks. only_results : boolean, optional Indicates whether output PDF should contain only result related info. Set to `False` as default. When set to `True`, the PDF will have info @@ -714,7 +704,7 @@ def generate_PDF_report(html_report_path=None, username="", password=None, goto="temp") return filepath, valid_result_uuids -def generate_score_matrix(username="", password=None, environment="production", model_list=[], model_instance_list=[], test_list=[], test_instance_list=[], result_list=[], project_id=None, client_obj=None): +def generate_score_matrix(username="", password=None, environment="production", model_list=[], model_instance_list=[], test_list=[], test_instance_list=[], result_list=[], show_links=True, client_obj=None): """Generates a styled pandas dataframe with score matrix This method will generate a styled pandas dataframe for the specified test results. @@ -741,9 +731,9 @@ def generate_score_matrix(username="", password=None, environment="production", List of test instance UUIDs for which score matrix is to be generated. result_list : list List of result UUIDs for which score matrix is to be generated. - project_id : string, optional - Collaboratory ID where hyperlinks to results are to be redirected. - If unspecified, the scores will not have clickable hyperlinks. + show_links : boolean, optional + To specify if hyperlinks to results are to be provided. + If false, these data units will not have clickable hyperlinks. client_obj : ModelCatalog/TestLibrary object Used to easily create a new ModelCatalog/TestLibrary object if either exist already. Avoids need for repeated authentications; improves performance. Also, helps minimize @@ -785,11 +775,6 @@ def generate_score_matrix(username="", password=None, environment="production", else: test_library = TestLibrary(username, password, environment=environment) - if project_id: - # check if app exists; if not then create - VFapp_navID = test_library.exists_in_collab_else_create(project_id) - test_library.set_app_config(project_id=project_id, app_id=VFapp_navID, only_if_new="True") - # retrieve all model instances from specified models if model_list: for entry in model_list: @@ -886,8 +871,8 @@ def make_clickable(value): if not value: return value score, result_uuid = value.split('#*#') - if project_id: - result_url = "https://collab.humanbrainproject.eu/#/collab/{}/nav/{}?state=result.{}".format(str(project_id),str(VFapp_navID), result_uuid) + if show_links: + result_url = "https://model-catalog.brainsimulation.eu/#result_id.{}".format(result_uuid) return '{}'.format(result_url,score) else: return score diff --git a/setup.py b/setup.py index 67d738f..c7e9122 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def package_files(base_dir, directory): setup( name='hbp_validation_framework', - version='0.6.0', + version='0.6.1', packages=['hbp_validation_framework'], package_data={'': json_files+template_files}, url='https://github.com/HumanBrainProject/hbp-validation-client', diff --git a/test.txt b/test.txt deleted file mode 100644 index ce01362..0000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -hello diff --git a/tests/conftest.py b/tests/conftest.py index 06719a5..db81ad8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def myModelID(modelCatalog): model_catalog = modelCatalog model_name = "Model_{}_{}_py{}".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) # todo: need to test with both "single cell" and "network" as model_scope, since these have different KG representations - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -66,20 +66,20 @@ def myModelID(modelCatalog): "caption":"NEURON Logo"}, {"url":"https://raw.githubusercontent.com/HumanBrainProject/hbp-validation-client/master/eu_logo.jpg", "caption":"HBP Logo"}]) - return model_id + return model["id"] @pytest.fixture(scope="session") def myTestID(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") - isinstance_id = testLibrary.add_test_instance(test_id=test_id, version="2.0", repository="http://www.12345.com", path="hbp_validation_framework.sample.SampleTest", parameters="", description="") - return test_id + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) + isinstance_id = testLibrary.add_test_instance(test_id=test["id"], version="2.0", repository="http://www.12345.com", path="hbp_validation_framework.sample.SampleTest", parameters="", description="") + return test["id"] @pytest.fixture(scope="session") @@ -93,27 +93,27 @@ def myResultID(modelCatalog, testLibrary, myModelID, myTestID): model = sample.SampleModel(model_uuid=model_id, model_version=model["instances"][0]["version"]) test_name = "Test_{}_{}_py{}_getValTest_1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) sleep(20) - test = test_library.get_validation_test(test_id=test_id) + test = test_library.get_validation_test(test_id=test["id"]) score = test.judge(model) timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") folder_name = "results_{}_{}_{}".format(model.name, model.model_uuid[:8], timestamp) - result_id = test_library.register_result(score, project_id="model-validation") # Collab ID for testing - return result_id + result = test_library.register_result(score, collab_id="model-validation") # Collab ID for testing + return result["id"] def pytest_sessionfinish(session, exitstatus): ENVIRONMENT = session.config.getoption("--environment") model_catalog = ModelCatalog(username=HBP_USERNAME, password=HBP_PASSWORD, environment=ENVIRONMENT) # model_catalog = ModelCatalog(token=TOKEN, environment=ENVIRONMENT) - models = model_catalog.list_models(project_id="model-validation", author={"family_name": "Tester", "given_name": "Validation"}) + models = model_catalog.list_models(collab_id="model-validation", author={"family_name": "Tester", "given_name": "Validation"}) for model in models: if "IGNORE - Test Model - " in model["name"]: model_catalog.delete_model(model["id"]) diff --git a/tests/test_model_instances.py b/tests/test_model_instances.py index 5481e40..6057bd0 100644 --- a/tests/test_model_instances.py +++ b/tests/test_model_instances.py @@ -114,7 +114,7 @@ def test_addModelInstance_valid(modelCatalog, myModelID): hash="", morphology="", description="") - assert isinstance(uuid.UUID(model_instance, version=4), uuid.UUID) + assert isinstance(uuid.UUID(model_instance["id"], version=4), uuid.UUID) #3.2) With no model_id def test_addModelInstance_no_id(modelCatalog): @@ -127,7 +127,7 @@ def test_addModelInstance_no_id(modelCatalog): hash="", morphology="", description="") - assert str(excinfo.value) == "Model ID needs to be provided for finding the model." + assert str(excinfo.value) == "model_id or alias needs to be provided for finding the model." #3.3) With invalid model_id format def test_addModelInstance_invalid_id_format(modelCatalog): @@ -192,15 +192,14 @@ def test_editModelInstance_valid_id(modelCatalog, myModelID): model_id = myModelID sleep(20) model = model_catalog.get_model(model_id=model_id) - model_instance_id = model_catalog.edit_model_instance(instance_id=model["instances"][0]["id"], + model_instance = model_catalog.edit_model_instance(instance_id=model["instances"][0]["id"], source="https://www.abcde.com", parameters="a", code_format="b", hash="c", morphology="http://example.com/d.txt", description="e") - assert model_instance_id == model["instances"][0]["id"] - model_instance = model_catalog.get_model_instance(instance_id=model_instance_id) + assert model_instance["id"] == model["instances"][0]["id"] assert model_instance["source"] == "https://www.abcde.com" assert model_instance["parameters"] == "a" assert model_instance["code_format"] == "b" @@ -214,15 +213,14 @@ def test_editModelInstance_valid_model_version(modelCatalog, myModelID): model_id = myModelID sleep(20) model = model_catalog.get_model(model_id=model_id) - model_instance_id = model_catalog.edit_model_instance(model_id=model_id, version=model["instances"][0]["version"], + model_instance = model_catalog.edit_model_instance(model_id=model_id, version=model["instances"][0]["version"], source="https://www.abcde.com", parameters="a", code_format="b", hash="c", morphology="http://example.com/d.txt", description="e") - assert model_instance_id == model["instances"][0]["id"] - model_instance = model_catalog.get_model_instance(instance_id=model_instance_id) + assert model_instance["id"] == model["instances"][0]["id"] assert model_instance["source"] == "https://www.abcde.com" assert model_instance["parameters"] == "a" assert model_instance["code_format"] == "b" @@ -236,7 +234,7 @@ def test_editModelInstance_valid_alias_version(modelCatalog, myModelID): model_id = myModelID model = model_catalog.get_model(model_id=model_id) sleep(20) - model_instance_id = model_catalog.edit_model_instance(alias=model["alias"], + model_instance = model_catalog.edit_model_instance(alias=model["alias"], version=model["instances"][0]["version"], source="https://www.abcde.com", parameters="a", @@ -244,8 +242,7 @@ def test_editModelInstance_valid_alias_version(modelCatalog, myModelID): hash="c", morphology="http://example.com/d.txt", description="e") - assert model_instance_id == model["instances"][0]["id"] - model_instance = model_catalog.get_model_instance(instance_id=model_instance_id) + assert model_instance["id"] == model["instances"][0]["id"] assert model_instance["source"] == "https://www.abcde.com" assert model_instance["parameters"] == "a" assert model_instance["code_format"] == "b" @@ -304,7 +301,7 @@ def test_editModelInstance_valid_change_version(modelCatalog, myModelID): sleep(20) model = model_catalog.get_model(model_id=model_id) model_instance = model_catalog.get_model_instance(model_id=model_id, version="1.0a") - model_instance_id = model_catalog.edit_model_instance(instance_id=model_instance["id"], + model_instance = model_catalog.edit_model_instance(instance_id=model_instance["id"], version="a.1") assert "1.0a" not in [i["id"] for i in model["instances"]] @@ -366,8 +363,8 @@ def test_findCreateModelInstance_valid_exist_instance_id(modelCatalog, myModelID model_id = myModelID model = model_catalog.get_model(model_id=model_id) test_model = sample.SampleModel(model_instance_uuid=model["instances"][0]["id"]) - model_instance_id = model_catalog.find_model_instance_else_add(test_model) - assert isinstance(uuid.UUID(model_instance_id, version=4), uuid.UUID) + model_instance = model_catalog.find_model_instance_else_add(test_model) + assert isinstance(uuid.UUID(model_instance["id"], version=4), uuid.UUID) #6.2) With valid details - existing model id and version def test_findCreateModelInstance_valid_exist_modelID_version(modelCatalog, myModelID): @@ -375,13 +372,13 @@ def test_findCreateModelInstance_valid_exist_modelID_version(modelCatalog, myMod model_id = myModelID model = model_catalog.get_model(model_id=model_id) test_model = sample.SampleModel(model_uuid=model_id, model_version=model["instances"][0]["version"]) - model_instance_id = model_catalog.find_model_instance_else_add(test_model) - assert isinstance(uuid.UUID(model_instance_id, version=4), uuid.UUID) + model_instance = model_catalog.find_model_instance_else_add(test_model) + assert isinstance(uuid.UUID(model_instance["id"], version=4), uuid.UUID) def test_findCreateModelInstance_valid_create(modelCatalog, myModelID): model_catalog = modelCatalog model_id = myModelID model = model_catalog.get_model(model_id=model_id) test_model = sample.SampleModel(model_uuid=model_id, model_version=model["instances"][0]["version"]+"_new") - model_instance_id = model_catalog.find_model_instance_else_add(test_model) - assert isinstance(uuid.UUID(model_instance_id, version=4), uuid.UUID) + model_instance = model_catalog.find_model_instance_else_add(test_model) + assert isinstance(uuid.UUID(model_instance["id"], version=4), uuid.UUID) diff --git a/tests/test_models.py b/tests/test_models.py index ddbaa76..6e56657 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -106,12 +106,12 @@ def test_getList_no_filter(modelCatalog): #2.2) Single filter def test_getList_one_filter(modelCatalog, myModelID): model_catalog = modelCatalog - models = model_catalog.list_models(project_id="model-validation") + models = model_catalog.list_models(collab_id="model-validation") assert isinstance(models, list) assert len(models) > 0 - project_ids = set([model["project_id"] for model in models]) - assert len(project_ids) == 1 - value, = project_ids + collab_ids = set([model["collab_id"] for model in models]) + assert len(collab_ids) == 1 + value, = collab_ids assert value == "model-validation" #2.3) Multiple filters @@ -190,7 +190,7 @@ def test_getModelValid_invalid(modelCatalog): def test_addModel_none(modelCatalog): model_catalog = modelCatalog with pytest.raises(Exception) as excinfo: - model_id = model_catalog.register_model() + model = model_catalog.register_model() #4.2) Missing mandatory parameter (model name) # Note: author name was no longer mandatory, so changed to model name @@ -198,7 +198,7 @@ def test_addModel_none(modelCatalog): def test_addModel_missingParam(modelCatalog): model_catalog = modelCatalog with pytest.raises(Exception) as excinfo: - model_id = model_catalog.register_model(project_id="model-validation", organization="HBP-SP6", + model = model_catalog.register_model(collab_id="model-validation", organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", brain_region="basal ganglia", species="Mus musculus", @@ -211,7 +211,7 @@ def test_addModel_invalidParam(modelCatalog): model_catalog = modelCatalog with pytest.raises(Exception) as excinfo: model_name = "Model_{}_{}_py{}_add3".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -224,33 +224,33 @@ def test_addModel_invalidParam(modelCatalog): def test_addModel_valid_noalias_nodetails(modelCatalog): model_catalog = modelCatalog model_name = "Model_{}_{}_py{}_add4".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", brain_region="basal ganglia", species="Mus musculus", owner={"family_name": "Tester", "given_name": "Validation"}, license="BSD 3-Clause", description="This is a test entry! Please ignore.") - assert isinstance(uuid.UUID(model_id, version=4), uuid.UUID) + assert isinstance(uuid.UUID(model["id"], version=4), uuid.UUID) #4.5) Valid model with alias; without instances and images def test_addModel_valid_withalias_nodetails(modelCatalog): model_catalog = modelCatalog model_name = "Model_{}_{}_py{}_add5".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", brain_region="basal ganglia", species="Mus musculus", owner={"family_name": "Tester", "given_name": "Validation"}, license="BSD 3-Clause", description="This is a test entry! Please ignore.") - assert isinstance(uuid.UUID(model_id, version=4), uuid.UUID) + assert isinstance(uuid.UUID(model["id"], version=4), uuid.UUID) #4.6) Invalid model with repeated alias; without instances and images def test_addModel_repeat_alias_nodetails(modelCatalog): model_catalog = modelCatalog model_name = "Model_{}_{}_py{}_add6".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -259,7 +259,7 @@ def test_addModel_repeat_alias_nodetails(modelCatalog): description="This is a test entry! Please ignore.") sleep(30) with pytest.raises(Exception) as excinfo: - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -273,7 +273,7 @@ def test_addModel_repeat_alias_nodetails(modelCatalog): def test_addModel_valid_withalias_withdetails(modelCatalog): model_catalog = modelCatalog model_name = "Model_{}_{}_py{}_add7".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -288,7 +288,7 @@ def test_addModel_valid_withalias_withdetails(modelCatalog): "caption":"NEURON Logo"}, {"url":"https://collab.humanbrainproject.eu/assets/hbp_diamond_120.png", "caption":"HBP Logo"}]) - assert isinstance(uuid.UUID(model_id, version=4), uuid.UUID) + assert isinstance(uuid.UUID(model["id"], version=4), uuid.UUID) """ @@ -299,17 +299,16 @@ def test_addModel_valid_withalias_withdetails(modelCatalog): def test_editModel_invalid_noID(modelCatalog): model_catalog = modelCatalog model_name = "Model_{}_{}_py{}_edit1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", brain_region="basal ganglia", species="Mus musculus", owner={"family_name": "Tester", "given_name": "Validation"}, license="BSD 3-Clause", description="This is a test entry! Please ignore.") - model = model_catalog.get_model(model_id=model_id) with pytest.raises(Exception) as excinfo: - model_id = model_catalog.edit_model( - project_id="model-validation", name=model["name"] + "_changed", + model = model_catalog.edit_model( + collab_id="model-validation", name=model["name"] + "_changed", alias = model["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", @@ -323,29 +322,28 @@ def test_editModel_invalid_noID(modelCatalog): def test_editModel_valid(modelCatalog): model_catalog = modelCatalog model_name = "Model_{}_{}_py{}_edit2".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", brain_region="basal ganglia", species="Mus musculus", owner={"family_name": "Tester", "given_name": "Validation"}, license="BSD 3-Clause", description="This is a test entry! Please ignore.") - model = model_catalog.get_model(model_id=model_id) - model_id = model_catalog.edit_model(model_id=model_id, - project_id="model-validation", name=model["name"] + "_changed", + model = model_catalog.edit_model(model_id=model["id"], + collab_id="model-validation", name=model["name"] + "_changed", alias = model["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP4", private=False, cell_type="pyramidal cell", model_scope="network: whole brain", abstraction_level="systems biology", brain_region="hippocampus", species="Rattus norvegicus", owner={"family_name": "Tester", "given_name": "Validation"}, license="BSD 2-Clause", description="This is a test entry! Please ignore.") - assert isinstance(uuid.UUID(model_id, version=4), uuid.UUID) + assert isinstance(uuid.UUID(model["id"], version=4), uuid.UUID) #5.3) Invalid change - duplicate alias def test_editModel_invalid_duplicate_alias(modelCatalog): model_catalog = modelCatalog model_name1 = "Model_{}_{}_py{}_edit3.1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name1, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name1, alias=model_name1, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -353,7 +351,7 @@ def test_editModel_invalid_duplicate_alias(modelCatalog): owner={"family_name": "Tester", "given_name": "Validation"}, license="BSD 3-Clause", description="This is a test entry! Please ignore.") model_name2 = "Model_{}_{}_py{}_edit3.2".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name2, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name2, alias=model_name2, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -361,10 +359,10 @@ def test_editModel_invalid_duplicate_alias(modelCatalog): owner={"family_name": "Tester", "given_name": "Validation"}, license="BSD 3-Clause", description="This is a test entry! Please ignore.") sleep(20) - model = model_catalog.get_model(model_id=model_id) + model = model_catalog.get_model(model_id=model["id"]) with pytest.raises(Exception) as excinfo: - model_id = model_catalog.edit_model(model_id=model_id, - project_id="model-validation", name=model["name"] + "_changed", + model = model_catalog.edit_model(model_id=model["id"], + collab_id="model-validation", name=model["name"] + "_changed", alias = model_name1, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", diff --git a/tests/test_results.py b/tests/test_results.py index 9ba0698..9a59a06 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -24,22 +24,22 @@ def test_register_result_valid(modelCatalog, testLibrary, myModelID, myTestID): model = sample.SampleModel(model_uuid=model_id, model_version=model["instances"][0]["version"]) test_name = "Test_{}_{}_py{}_getValTest_1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) sleep(20) - test = test_library.get_validation_test(test_id=test_id) + test = test_library.get_validation_test(test_id=test["id"]) score = test.judge(model) timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") folder_name = "results_{}_{}_{}".format(model.name, model.model_uuid[:8], timestamp) - result_id = test_library.register_result(score, project_id="model-validation") - assert isinstance(uuid.UUID(result_id, version=4), uuid.UUID) + result = test_library.register_result(score, collab_id="model-validation") + assert isinstance(uuid.UUID(result["id"], version=4), uuid.UUID) """ diff --git a/tests/test_superuser.py b/tests/test_superuser.py index ae80a68..68cb63c 100644 --- a/tests/test_superuser.py +++ b/tests/test_superuser.py @@ -31,7 +31,7 @@ def test_delete_superUser(request): raise Exception("Credentials not provided. Please define environment variables (HBP_AUTH_TOKEN or HBP_USER and HBP_PASS") model_name = "Model_{}_{}_py{}_superuser1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="model-validation", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="model-validation", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -43,34 +43,32 @@ def test_delete_superUser(request): images=[{"url":"http://www.neuron.yale.edu/neuron/sites/default/themes/xchameleon/logo.png", "caption":"NEURON Logo"}]) - model = model_catalog.get_model(model_id=model_id) model_instance_id = model["instances"][0]["id"] #model_image_id = model["images"][0]["id"] - model = sample.SampleModel(model_uuid=model_id, model_version=model["instances"][0]["version"]) + model_obj = sample.SampleModel(model_uuid=model["id"], model_version=model["instances"][0]["version"]) test_library = TestLibrary.from_existing(model_catalog) test_name = "Test_{}_{}_py{}_superuser2".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) - sleep(20) - test = test_library.get_test_definition(test_id=test_id) test_instance_id = test["instances"][0]["id"] - test = test_library.get_validation_test(test_id=test_id) + sleep(20) + test_obj = test_library.get_validation_test(test_id=test["id"]) - score = test.judge(model) + score = test_obj.judge(model_obj) timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") - folder_name = "results_{}_{}_{}".format(model.name, model.model_uuid[:8], timestamp) - result_id = test_library.register_result(score, project_id = "model-validation") # Collab ID = model-validation + folder_name = "results_{}_{}_{}".format(model_obj.name, model_obj.model_uuid[:8], timestamp) + result = test_library.register_result(score, collab_id = "model-validation") # Collab ID = model-validation - test_library.delete_result(result_id=result_id) + test_library.delete_result(result_id=result["id"]) sleep(20) with pytest.raises(Exception) as excinfo: - result = test_library.get_result(result_id=result_id) + result = test_library.get_result(result_id=result["id"]) assert "not found" in str(excinfo.value) model_catalog.delete_model_instance(instance_id=model_instance_id) @@ -79,10 +77,10 @@ def test_delete_superUser(request): model_instance = model_catalog.get_model_instance(instance_id=model_instance_id) assert "Error in retrieving model instance." in str(excinfo.value) - model_catalog.delete_model(model_id=model_id) + model_catalog.delete_model(model_id=model["id"]) sleep(20) with pytest.raises(Exception) as excinfo: - model = model_catalog.get_model(model_id=model_id) + model = model_catalog.get_model(model_id=model["id"]) assert "Error in retrieving model." in str(excinfo.value) test_library.delete_test_instance(instance_id=test_instance_id) @@ -91,10 +89,10 @@ def test_delete_superUser(request): test_instance = test_library.get_test_instance(instance_id=test_instance_id) assert "Error in retrieving test instance." in str(excinfo.value) - test_library.delete_test(test_id=test_id) + test_library.delete_test(test_id=test["id"]) sleep(20) with pytest.raises(Exception) as excinfo: - test = test_library.get_test_definition(test_id=test_id) + test = test_library.get_test_definition(test_id=test["id"]) assert "Error in retrieving test" in str(excinfo.value) @@ -110,7 +108,7 @@ def test_delete_normalUser(request): else: raise Exception("Credentials not provided. Please define environment variables (HBP_AUTH_TOKEN or HBP_USER and HBP_PASS") model_name = "Model_{}_{}_py{}_normaluser1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), model_catalog.environment, platform.python_version()) - model_id = model_catalog.register_model(project_id="validation-tester", name="IGNORE - Test Model - " + model_name, + model = model_catalog.register_model(collab_id="validation-tester", name="IGNORE - Test Model - " + model_name, alias=model_name, author={"family_name": "Tester", "given_name": "Validation"}, organization="HBP-SP6", private=False, cell_type="granule cell", model_scope="single cell", abstraction_level="spiking neurons", @@ -122,40 +120,39 @@ def test_delete_normalUser(request): images=[{"url":"http://www.neuron.yale.edu/neuron/sites/default/themes/xchameleon/logo.png", "caption":"NEURON Logo"}]) - model = model_catalog.get_model(model_id=model_id) model_instance_id = model["instances"][0]["id"] #model_image_id = model["images"][0]["id"] - model = sample.SampleModel(model_uuid=model_id, model_version=model["instances"][0]["version"]) + model_obj = sample.SampleModel(model_uuid=model["id"], model_version=model["instances"][0]["version"]) test_library = TestLibrary.from_existing(model_catalog) test_name = "Test_{}_{}_py{}_normaluser2".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") - sleep(20) - test = test_library.get_test_definition(test_id=test_id) + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) + test_instance_id = test["instances"][0]["id"] - test = test_library.get_validation_test(test_id=test_id) + sleep(20) + test_obj = test_library.get_validation_test(test_id=test["id"]) - score = test.judge(model) + score = test_obj.judge(model_obj) timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") - folder_name = "results_{}_{}_{}".format(model.name, model.model_uuid[:8], timestamp) + folder_name = "results_{}_{}_{}".format(model_obj.name, model_obj.model_uuid[:8], timestamp) - result_id = test_library.register_result(score, project_id="validation-tester") # Collab ID = validation-tester + result = test_library.register_result(score, collab_id="validation-tester") # Collab ID = validation-tester # normal users cannot delete results with pytest.raises(Exception) as excinfo: - test_library.delete_result(result_id=result_id) + test_library.delete_result(result_id=result["id"]) assert "Only SuperUser accounts can delete data." in str(excinfo.value) # normal users can delete model instances that they have write access to model_catalog.delete_model_instance(instance_id=model_instance_id) # normal users can delete models that they have write access to - model_catalog.delete_model(model_id=model_id) + model_catalog.delete_model(model_id=model["id"]) # normal users cannot delete test instances with pytest.raises(Exception) as excinfo: @@ -164,5 +161,5 @@ def test_delete_normalUser(request): # normal users cannot delete tests with pytest.raises(Exception) as excinfo: - test_library.delete_test(test_id=test_id) + test_library.delete_test(test_id=test["id"]) assert "Only SuperUser accounts can delete data." in str(excinfo.value) diff --git a/tests/test_test_instances.py b/tests/test_test_instances.py index a4367d6..aa20c5c 100644 --- a/tests/test_test_instances.py +++ b/tests/test_test_instances.py @@ -108,7 +108,7 @@ def test_addTestInstance_valid(testLibrary, myTestID): path="hbp_validation_framework.sample.SampleTest", parameters="", description="") - assert isinstance(uuid.UUID(test_instance, version=4), uuid.UUID) + assert isinstance(uuid.UUID(test_instance["id"], version=4), uuid.UUID) #3.2) With no test_id def test_addTestInstance_no_id(testLibrary): @@ -172,13 +172,12 @@ def test_editTestInstance_valid_id(testLibrary, myTestID): test_id = myTestID sleep(20) test = test_library.get_test_definition(test_id=test_id) - test_instance_id = test_library.edit_test_instance(instance_id=test["instances"][0]["id"], + test_instance = test_library.edit_test_instance(instance_id=test["instances"][0]["id"], repository="http://www.12345.com", path="hbp_validation_framework.sample.SampleTest", parameters="d", description="e") - assert test_instance_id == test["instances"][0]["id"] - test_instance = test_library.get_test_instance(instance_id=test_instance_id) + assert test_instance["id"] == test["instances"][0]["id"] assert test_instance["repository"] == "http://www.12345.com" assert test_instance["path"] == "hbp_validation_framework.sample.SampleTest" assert test_instance["parameters"] == "d" @@ -190,14 +189,12 @@ def test_editTestInstance_valid_test_version(testLibrary, myTestID): test_id = myTestID sleep(20) test = test_library.get_test_definition(test_id=test_id) - test_instance_id = test_library.edit_test_instance(test_id=test_id, version=test["instances"][0]["version"], + test_instance = test_library.edit_test_instance(test_id=test_id, version=test["instances"][0]["version"], repository="https://www.12345.com", path="hbp_validation_framework.sample.SampleTest", parameters="d", description="e") - assert test_instance_id == test["instances"][0]["id"] - sleep(20) - test_instance = test_library.get_test_instance(instance_id=test_instance_id) + assert test_instance["id"] == test["instances"][0]["id"] assert test_instance["repository"] == "https://www.12345.com" assert test_instance["path"] == "hbp_validation_framework.sample.SampleTest" assert test_instance["parameters"] == "d" @@ -209,14 +206,12 @@ def test_editTestInstance_valid_alias_version(testLibrary, myTestID): test_id = myTestID sleep(20) test = test_library.get_test_definition(test_id=test_id) - test_instance_id = test_library.edit_test_instance(alias=test["alias"], version=test["instances"][0]["version"], + test_instance = test_library.edit_test_instance(alias=test["alias"], version=test["instances"][0]["version"], repository="https://www.abcde.com", path="hbp_validation_framework.sample.SampleTest", parameters="d", description="e") - assert test_instance_id == test["instances"][0]["id"] - sleep(20) - test_instance = test_library.get_test_instance(instance_id=test_instance_id) + assert test_instance["id"] == test["instances"][0]["id"] assert test_instance["repository"] == "https://www.abcde.com" assert test_instance["path"] == "hbp_validation_framework.sample.SampleTest" assert test_instance["parameters"] == "d" @@ -265,12 +260,12 @@ def test_editTestInstance_valid_change_version(testLibrary, myTestID): test_library = testLibrary test_id = myTestID - test_instance_id = test_library.add_test_instance(test_id=test_id, version="1.0_edit", + test_instance = test_library.add_test_instance(test_id=test_id, version="1.0_edit", repository="http://www.12345.com", path="hbp_validation_framework.sample.SampleTest", parameters="", description="") - test_instance_id = test_library.edit_test_instance(instance_id=test_instance_id, + test_instance = test_library.edit_test_instance(instance_id=test_instance["id"], version="a.1_edit") sleep(20) test_instances = test_library.list_test_instances(test_id=test_id) diff --git a/tests/test_tests.py b/tests/test_tests.py index 6821001..9dda30c 100644 --- a/tests/test_tests.py +++ b/tests/test_tests.py @@ -173,19 +173,19 @@ def test_getTestValid_invalid(testLibrary): def test_addtest_none(testLibrary): test_library = testLibrary with pytest.raises(Exception) as excinfo: - test_id = test_library.add_test() + test = test_library.add_test() #4.2) Missing mandatory parameter (author) def test_addtest_missingParam(testLibrary): test_library = testLibrary with pytest.raises(Exception) as excinfo: test_name = "Test_{}_{}_py{}_add2".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) assert "field required" in str(excinfo.value) #4.3) Invalid value for parameter (brain_region) @@ -193,55 +193,55 @@ def test_addtest_invalidParam(testLibrary): test_library = testLibrary with pytest.raises(Exception) as excinfo: test_name = "Test_{}_{}_py{}_add3".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="ABCDE", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) assert "brain_region = 'ABCDE' is invalid." in str(excinfo.value) #4.4) Valid test without alias def test_addtest_valid_noalias_nodetails(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_add4".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) #4.5) Valid test with alias def test_addtest_valid_withalias_nodetails(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_add5".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") - assert isinstance(uuid.UUID(test_id, version=4), uuid.UUID) + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) + assert isinstance(uuid.UUID(test["id"], version=4), uuid.UUID) #4.6) Invalid test with repeated alias; without instances and images def test_addtest_repeat_alias_nodetails(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_add6".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) sleep(20) with pytest.raises(Exception) as excinfo: - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) assert "already exists" in str(excinfo.value) #4.7) Invalid test with no instances @@ -249,9 +249,9 @@ def test_addtest_valid_withalias_withdetails(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_add7".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) with pytest.raises(Exception) as excinfo: - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019") assert "Error in adding test." in str(excinfo.value) @@ -265,17 +265,16 @@ def test_addtest_valid_withalias_withdetails(testLibrary): def test_editTest_invalid_noID(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_edit1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") - test = test_library.get_test_definition(test_id=test_id) + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) with pytest.raises(Exception) as excinfo: - test_id = test_library.edit_test(name="IGNORE - Test Test - " + test_name, alias=test["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.edit_test(name="IGNORE - Test Test - " + test_name, alias=test["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019") assert str(excinfo.value) == "Test ID needs to be provided for editing a test." @@ -284,43 +283,42 @@ def test_editTest_invalid_noID(testLibrary): def test_editTest_valid(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_edit2".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") - test = test_library.get_test_definition(test_id=test_id) - test_id = test_library.edit_test(test_id=test_id, name="IGNORE - Test Test - " + test_name, alias=test["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) + test = test_library.edit_test(test_id=test["id"], name="IGNORE - Test Test - " + test_name, alias=test["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019") - assert isinstance(uuid.UUID(test_id, version=4), uuid.UUID) + assert isinstance(uuid.UUID(test["id"], version=4), uuid.UUID) #5.3) Invalid change - duplicate alias def test_editTest_invalid_duplicate_alias(testLibrary): test_library = testLibrary test_name1 = "Test_{}_{}_py{}_edit3.1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name1, alias=test_name1, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name1, alias=test_name1, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) test_name2 = "Test_{}_{}_py{}_edit3.2".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name2, alias=test_name2, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name2, alias=test_name2, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) sleep(20) - test = test_library.get_test_definition(test_id=test_id) + test = test_library.get_test_definition(test_id=test["id"]) with pytest.raises(Exception) as excinfo: - test_id = test_library.edit_test(test_id=test_id, name=test["name"] + "_changed", alias=test_name1, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.edit_test(test_id=test["id"], name=test["name"] + "_changed", alias=test_name1, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019") assert "already exists" in str(excinfo.value) @@ -329,21 +327,21 @@ def test_editTest_invalid_duplicate_alias(testLibrary): def test_editTest_invalid_version_info(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_edit4".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") - test = test_library.get_test_definition(test_id=test_id) + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) + test = test_library.get_test_definition(test_id=test["id"]) with pytest.raises(Exception) as excinfo: - test_id = test_library.edit_test(test_id=test_id, name="IGNORE - Test Test - " + test_name, alias=test["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, + test["id"] = test_library.edit_test(test_id=test["id"], name="IGNORE - Test Test - " + test_name, alias=test["alias"] + "_changed", author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") - assert "got an unexpected keyword argument 'version'" in str(excinfo.value) + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) + assert "got an unexpected keyword argument" in str(excinfo.value) """ @@ -354,13 +352,13 @@ def test_editTest_invalid_version_info(testLibrary): def test_getValidationTest_testID(testLibrary): test_library = testLibrary test_name = "Test_{}_{}_py{}_getValTest_1".format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"), test_library.environment, platform.python_version()) - test_id = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, + test = test_library.add_test(name="IGNORE - Test Test - " + test_name, alias=test_name, author={"family_name": "Tester", "given_name": "Validation"}, species="Mus musculus", age="", brain_region="basal ganglia", cell_type="granule cell", - recording_modality="electron microscopy", test_type="network structure", score_type="Other", protocol="Later", + recording_modality="electron microscopy", test_type="network structure", score_type="Other", description="Later", data_location="https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/sp6_validation_data/test.txt", data_type="Mean, SD", publication="Testing et al., 2019", - version="1.0", repository="https://github.com/HumanBrainProject/hbp-validation-client.git", path="hbp_validation_framework.sample.SampleTest") + instances=[{"version":"1.0", "repository":"https://github.com/HumanBrainProject/hbp-validation-client.git", "path":"hbp_validation_framework.sample.SampleTest"}]) sleep(30) - test = test_library.get_validation_test(test_id=test_id) + test = test_library.get_validation_test(test_id=test["id"]) assert isinstance(test, sciunit.Test) assert "test.txt" in test.observation[0] diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 6d78ff1..ac55369 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -78,8 +78,8 @@ def test_upload_test_result(modelCatalog, testLibrary, myModelID, myTestID): model = model_catalog.get_model(model_id=model_id) test_model = sample.SampleModel(model_instance_uuid=model["instances"][0]["id"]) test_result_file = utils.run_test_offline(model=test_model, test_config_file=test_config_file) - result_id, score = utils.upload_test_result(test_result_file=test_result_file, client_obj=test_library) - assert isinstance(uuid.UUID(result_id, version=4), uuid.UUID) + result, score = utils.upload_test_result(test_result_file=test_result_file, client_obj=test_library) + assert isinstance(uuid.UUID(result["id"], version=4), uuid.UUID) assert isinstance(score, sciunit.Score) @@ -97,6 +97,6 @@ def test_run_test_combined(modelCatalog, testLibrary, myModelID, myTestID): model = model_catalog.get_model(model_id=model_id) test_model = sample.SampleModel(model_instance_uuid=model["instances"][0]["id"]) - result_id, score = utils.run_test(model=test_model, test_id=test_id, test_version=test["instances"][0]["version"], client_obj=test_library) - assert isinstance(uuid.UUID(result_id, version=4), uuid.UUID) + result, score = utils.run_test(model=test_model, test_id=test_id, test_version=test["instances"][0]["version"], client_obj=test_library) + assert isinstance(uuid.UUID(result["id"], version=4), uuid.UUID) assert isinstance(score, sciunit.Score) From c3dc9f8a295c6279cabd35ad17de672be3f11a54 Mon Sep 17 00:00:00 2001 From: Shailesh Appukuttan Date: Thu, 19 Nov 2020 21:28:41 +0100 Subject: [PATCH 2/2] minor fix --- hbp_validation_framework/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbp_validation_framework/utils.py b/hbp_validation_framework/utils.py index b10ab08..7857dea 100644 --- a/hbp_validation_framework/utils.py +++ b/hbp_validation_framework/utils.py @@ -815,7 +815,7 @@ def generate_score_matrix(username="", password=None, environment="production", excluded_results = [] # not latest entry for a particular model instance and test instance combination for r_id in result_list: - result = test_library.get_result(result_id = r_id)["results"][0] + result = test_library.get_result(result_id = r_id) # '#*#' is used as separator between score and result UUID (latter used for constructing hyperlink) if result["test_instance_id"] in results_dict.keys(): if result["model_instance_id"] not in results_dict[result["test_instance_id"]].keys():