From a11180848711200eca595d3c677c5bab20181f2b Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Tue, 28 Nov 2023 17:44:12 -0500 Subject: [PATCH 1/3] Add Dataset Activity.creation_action to ENTITY.Dataset as creation_action_activity --- entity-api-spec.yaml | 3 +++ src/app.py | 6 ++++++ src/instance/app.cfg.example | 4 ++++ src/schema/provenance_schema.yaml | 4 ++++ src/schema/schema_manager.py | 4 ++-- src/schema/schema_neo4j_queries.py | 14 ++++++++++++++ 6 files changed, 33 insertions(+), 2 deletions(-) diff --git a/entity-api-spec.yaml b/entity-api-spec.yaml index 8fd04211..4f9665ee 100644 --- a/entity-api-spec.yaml +++ b/entity-api-spec.yaml @@ -807,6 +807,9 @@ components: dbgap_study_url: type: string description: 'A URL linking the dataset to the particular study on dbGap it belongs to' + creation_action_activity: + type: string + description: 'The associated action that represents the creation of that dataset' Upload: type: object properties: diff --git a/src/app.py b/src/app.py index 4cbc05e8..fcc99622 100644 --- a/src/app.py +++ b/src/app.py @@ -4669,6 +4669,12 @@ def query_target_entity(id, user_token): # Make a new query against neo4j entity_dict = schema_neo4j_queries.get_entity(neo4j_driver_instance, uuid) + # entity_type is found in shared_entity_properties in provenance_schema.yaml + if entity_dict.get('entity_type') == 'Dataset': + # Entities.Dataset already has a creation_action property which is processed in a different manner + # see provenance_schema.yaml and code in this file. + entity_dict['creation_action_activity'] =\ + schema_neo4j_queries.get_entity_creation_action_activity(neo4j_driver_instance, uuid) # The uuid exists via uuid-api doesn't mean it also exists in Neo4j if not entity_dict: diff --git a/src/instance/app.cfg.example b/src/instance/app.cfg.example index 0c55f5bd..839972dc 100644 --- a/src/instance/app.cfg.example +++ b/src/instance/app.cfg.example @@ -28,6 +28,10 @@ UUID_API_URL = 'http://uuid-api:8080' # Works regardless of the trailing slash INGEST_API_URL = 'https://ingest-api.dev.hubmapconsortium.org' +# URL for talking to Ontology API (default for DEV) +# Works regardless of the trailing slash +ONTOLOGY_API_URL = 'https://ontology-api.dev.hubmapconsortium.org' + # A list of URLs for talking to multiple Search API instances (default value used for docker deployment, no token needed) # Works regardless of the trailing slash / SEARCH_API_URL_LIST = ['http://search-api:8080'] diff --git a/src/schema/provenance_schema.yaml b/src/schema/provenance_schema.yaml index 05174972..3691a3f2 100644 --- a/src/schema/provenance_schema.yaml +++ b/src/schema/provenance_schema.yaml @@ -284,6 +284,10 @@ ENTITIES: description: "The activity that was performed." before_property_create_validators: - validate_creation_action + creation_action_activity: + type: string + transient: true + description: "The creation action associated with the output activity of the dataset" description: type: string description: "Free text description of the dataset" diff --git a/src/schema/schema_manager.py b/src/schema/schema_manager.py index 0cafdecf..0d66229d 100644 --- a/src/schema/schema_manager.py +++ b/src/schema/schema_manager.py @@ -110,7 +110,7 @@ def load_provenance_schema(valid_yaml_file): with open(valid_yaml_file) as file: schema_dict = yaml.safe_load(file) - logger.info("Schema yaml file loaded successfully") + logger.info(f"Provenance Schema yaml file loaded successfully from {valid_yaml_file} :)") return schema_dict @@ -1794,4 +1794,4 @@ def _create_request_headers(user_token): auth_header_name: auth_scheme + ' ' + user_token } - return headers_dict \ No newline at end of file + return headers_dict diff --git a/src/schema/schema_neo4j_queries.py b/src/schema/schema_neo4j_queries.py index 8da148b3..547a30f0 100644 --- a/src/schema/schema_neo4j_queries.py +++ b/src/schema/schema_neo4j_queries.py @@ -502,6 +502,20 @@ def get_entity_type(neo4j_driver, entity_uuid: str) -> str: return None +def get_entity_creation_action_activity(neo4j_driver, entity_uuid: str) -> str: + query: str = f"MATCH (ds {{uuid:'{entity_uuid}'}})<-[:ACTIVITY_OUTPUT]-(a:Activity) RETURN a.creation_action" + + logger.info("======get_entity_creation_action() query======") + logger.info(query) + + with neo4j_driver.session() as session: + record = session.read_transaction(execute_readonly_tx, query) + if record and len(record) == 1: + return record[0] + + return None + + """ Create or recreate one or more linkages (via Activity nodes) between the target entity node and the direct ancestor nodes in neo4j From 1dd09990c282b68569af6c72aea253e70be9798b Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Wed, 29 Nov 2023 15:04:06 -0500 Subject: [PATCH 2/3] Moved creation_action to an on_read_trigger --- entity-api-spec.yaml | 2 +- src/app.py | 6 ------ src/schema/provenance_schema.yaml | 5 +---- src/schema/schema_triggers.py | 14 ++++++++++++++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/entity-api-spec.yaml b/entity-api-spec.yaml index 4f9665ee..25d807b7 100644 --- a/entity-api-spec.yaml +++ b/entity-api-spec.yaml @@ -807,7 +807,7 @@ components: dbgap_study_url: type: string description: 'A URL linking the dataset to the particular study on dbGap it belongs to' - creation_action_activity: + creation_action: type: string description: 'The associated action that represents the creation of that dataset' Upload: diff --git a/src/app.py b/src/app.py index fcc99622..4cbc05e8 100644 --- a/src/app.py +++ b/src/app.py @@ -4669,12 +4669,6 @@ def query_target_entity(id, user_token): # Make a new query against neo4j entity_dict = schema_neo4j_queries.get_entity(neo4j_driver_instance, uuid) - # entity_type is found in shared_entity_properties in provenance_schema.yaml - if entity_dict.get('entity_type') == 'Dataset': - # Entities.Dataset already has a creation_action property which is processed in a different manner - # see provenance_schema.yaml and code in this file. - entity_dict['creation_action_activity'] =\ - schema_neo4j_queries.get_entity_creation_action_activity(neo4j_driver_instance, uuid) # The uuid exists via uuid-api doesn't mean it also exists in Neo4j if not entity_dict: diff --git a/src/schema/provenance_schema.yaml b/src/schema/provenance_schema.yaml index 3691a3f2..d7f7e1de 100644 --- a/src/schema/provenance_schema.yaml +++ b/src/schema/provenance_schema.yaml @@ -281,13 +281,10 @@ ENTITIES: type: string transient: true immutable: true + on_read_trigger: get_creation_action_activity description: "The activity that was performed." before_property_create_validators: - validate_creation_action - creation_action_activity: - type: string - transient: true - description: "The creation action associated with the output activity of the dataset" description: type: string description: "Free text description of the dataset" diff --git a/src/schema/schema_triggers.py b/src/schema/schema_triggers.py index a24f2e32..fdad79d7 100644 --- a/src/schema/schema_triggers.py +++ b/src/schema/schema_triggers.py @@ -1149,6 +1149,20 @@ def get_next_revision_uuid(property_key, normalized_type, user_token, existing_d return property_key, next_revision_uuid +def get_creation_action_activity(property_key, normalized_type, user_token, existing_data_dict, new_data_dict): + if 'uuid' not in existing_data_dict: + raise KeyError("Missing 'uuid' key in 'existing_data_dict' during calling 'get_creation_action_activity()' trigger method.") + + uuid: str = existing_data_dict['uuid'] + logger.info(f"Executing 'get_creation_action_activity()' trigger method on uuid: {uuid}") + + neo4j_driver_instance = schema_manager.get_neo4j_driver_instance() + creation_action_activity =\ + schema_neo4j_queries.get_entity_creation_action_activity(neo4j_driver_instance, uuid) + + return property_key, creation_action_activity + + """ Trigger event method to commit thumbnail file saved that were previously uploaded via ingest-api From 1bc7ed0ce5142370613344d3daac0d05c804f768 Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Fri, 1 Dec 2023 15:20:32 -0500 Subject: [PATCH 3/3] Remove updating the UBKG server in the example as it should have been since this service started using UBKG in the .cfg file --- src/instance/app.cfg.example | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/instance/app.cfg.example b/src/instance/app.cfg.example index 839972dc..0c55f5bd 100644 --- a/src/instance/app.cfg.example +++ b/src/instance/app.cfg.example @@ -28,10 +28,6 @@ UUID_API_URL = 'http://uuid-api:8080' # Works regardless of the trailing slash INGEST_API_URL = 'https://ingest-api.dev.hubmapconsortium.org' -# URL for talking to Ontology API (default for DEV) -# Works regardless of the trailing slash -ONTOLOGY_API_URL = 'https://ontology-api.dev.hubmapconsortium.org' - # A list of URLs for talking to multiple Search API instances (default value used for docker deployment, no token needed) # Works regardless of the trailing slash / SEARCH_API_URL_LIST = ['http://search-api:8080']