Merge pull request #365 from DESm1th/xnat_sharing

[ENH] Auto-share XNAT sessions based on REDCap contents

assets/config_templates/main_config.yml

@@ -140,6 +140,13 @@ ExportSettings:
                                  # metadata folder that will hold the username
                                  # and password for the XnatSource server.
                                  # Must be provided if XnatSource is set.
+# XnatDataSharing: <boolean>     # Whether or not certain XNAT sessions for
+                                 # a study will be shared within XNAT. Any
+                                 # value is interpreted as 'True', if unset
+                                 # will be False. Shared IDs are read from
+                                 # the configured redcap survey (i.e. you
+                                 # should also set 'RedcapSharedIdPrefix' if
+                                 # you set this).
 
 
 ###### REDCap Configuration ##############
@@ -171,6 +178,11 @@ ExportSettings:
 # RedcapRecordKey: <fieldname>   # The name of the field that contains the
                                  # unique record ID.
 # RedcapComments: <fieldname>    # The name of field that holds RA comments.
+# RedcapSharedIdPrefix: <prefix> # The prefix that will be used for any
+                                 # survey field that contains an alternate
+                                 # ID for the session. Optional. If XNAT
+                                 # sessions must be shared then
+                                 # 'XnatDataSharing' must be set as well.
 
 
 ###### Log Server Configuration ##############

bin/dm_link_shared_ids.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 """
 Finds REDCap records for the given study and if a session has multiple IDs
-(i.e. it is shared with another study) updates the dashboard and tries to
-create links from the exported original data to its pseudonyms.
+(i.e. it is shared with another study) shares the session on xnat, updates the
+dashboard and tries to create links from the exported original data to its
+pseudonyms.
 
 Usage:
     dm_link_shared_ids.py [options] <project>
@@ -33,6 +34,7 @@
 import datman.config
 import datman.scanid
 import datman.utils
+import datman.xnat
 
 import bin.dm_link_project_scans as link_scans
 import datman.dashboard as dashboard
@@ -69,17 +71,32 @@ def main():
         link_scans.logger.setLevel(logging.ERROR)
 
     config = datman.config.config(filename=site_config, study=project)
+    if uses_xnat_sharing(config):
+        xnat = datman.xnat.get_connection(config)
+    else:
+        xnat = None
 
     scan_complete_records = get_redcap_records(config, redcap_cred)
     for record in scan_complete_records:
         if not record.shared_ids:
             continue
-        make_links(record)
+        share_session(record, xnat_connection=xnat)
 
 
 def get_redcap_records(config, redcap_cred):
     token = get_token(config, redcap_cred)
     redcap_url = config.get_key('RedcapApi')
+    current_study = config.get_key('StudyTag')
+
+    record_id_field = get_setting(config, 'RedcapRecordKey')
+    subid_field = get_setting(config, 'RedcapSubj', default='par_id')
+    shared_prefix_field = get_setting(
+        config, 'RedcapSharedIdPrefix', default='shared_parid')
+
+    try:
+        id_map = config.get_key('IdMap')
+    except UndefinedSetting:
+        id_map = None
 
     logger.debug("Accessing REDCap API at {}".format(redcap_url))
 
@@ -93,15 +110,10 @@ def get_redcap_records(config, redcap_cred):
         logger.error("Cannot access redcap data at URL {}".format(redcap_url))
         sys.exit(1)
 
-    current_study = config.get_key('StudyTag')
-
     try:
-        id_map = config.get_key('IdMap')
-    except UndefinedSetting:
-        id_map = None
-
-    try:
-        project_records = parse_records(response, current_study, id_map)
+        project_records = parse_records(
+            response, current_study, id_map,
+            record_id_field, subid_field, shared_prefix_field)
     except ValueError as e:
         logger.error("Couldn't parse redcap records for server response {}. "
                      "Reason: {}".format(response.content, e))
@@ -126,10 +138,24 @@ def get_token(config, redcap_cred):
     return token
 
 
-def parse_records(response, study, id_map):
+def get_setting(config, key, default=None):
+    """Get the REDCap survey field names, if they differ from a default.
+    """
+    try:
+        return config.get_key(key)
+    except datman.config.UndefinedSetting as e:
+        if not default:
+            raise e
+        return default
+
+
+def parse_records(response, study, id_map, record_id_field, id_field,
+                  shared_id_prefix_field):
     records = []
     for item in response.json():
-        record = Record(item, id_map)
+        record = Record(item, id_map, record_id_field=record_id_field,
+                        id_field=id_field,
+                        shared_id_prefix_field=shared_id_prefix_field)
         if record.id is None:
             logger.debug(
                 f"Record with ID {item['record_id']} has malformed subject ID "
@@ -141,13 +167,24 @@ def parse_records(response, study, id_map):
     return records
 
 
-def make_links(record):
+def uses_xnat_sharing(config):
+    """Check if source (xnat) data is to be shared also.
+    """
+    try:
+        config.get_key('XnatDataSharing')
+    except datman.config.UndefinedSetting:
+        return False
+    return True
+
+
+def share_session(record, xnat_connection=None):
     source = record.id
     for target in record.shared_ids:
         logger.info(
-            f"Making links from source {source} to target {target}"
+            f"Sharing source ID {source} under ID {target}"
         )
-        target_cfg = datman.config.config(study=target)
+
+        target_cfg = datman.config.config(study=str(target))
         try:
             target_tags = list(target_cfg.get_tags(site=source.site))
         except Exception:
@@ -157,14 +194,74 @@ def make_links(record):
 
         if DRYRUN:
             logger.info(
-                "DRYRUN - would have made links from source ID "
+                "DRYRUN - would have shared scans from source ID "
                 f"{source} to target ID {target} for tags {target_tags}"
             )
             continue
 
+        if xnat_connection:
+            share_xnat_data(xnat_connection, source, target, target_cfg)
+
         link_scans.create_linked_session(str(source), str(target), target_tags)
+
         if dashboard.dash_found:
-            share_redcap_record(target, record)
+            share_redcap_record(str(target), record)
+
+
+def share_xnat_data(xnat_connection, source, dest, config):
+    """Share an xnat subject/experiment under another ID.
+
+    Args:
+        xnat_connection (:obj:`datman.xnat.xnat`): A connection to the xnat
+            server containing the source data.
+        source (:obj:`datman.scanid.Identifier`): A datman Identifier for
+            the original session data.
+        dest (:obj:`datman.scanid.Identifier`): A datman Identifier for the
+            destination ID to share the data under.
+        config (:obj:`datman.config.config`): A datman configuration object.
+
+    Raises:
+        requests.HTTPError: If issues arise while communicating with the
+            server.
+    """
+    source_project = xnat_connection.find_project(
+        source.get_xnat_subject_id(),
+        config.get_xnat_projects(str(source))
+    )
+    dest_project = xnat_connection.find_project(
+        dest.get_xnat_subject_id(),
+        config.get_xnat_projects(str(dest))
+    )
+
+    try:
+        xnat_connection.share_subject(
+            source_project,
+            source.get_xnat_subject_id(),
+            dest_project,
+            dest.get_xnat_subject_id()
+        )
+    except datman.xnat.XnatException:
+        logger.info(f"XNAT subject {dest.get_xnat_subject_id()}"
+                    " already exists. No changes made.")
+    except requests.HTTPError as e:
+        logger.error(f"Failed while sharing source ID {source} into dest ID "
+                     f"{dest} on XNAT. Reason - {e}")
+        return
+
+    try:
+        xnat_connection.share_experiment(
+            source_project,
+            source.get_xnat_subject_id(),
+            source.get_xnat_experiment_id(),
+            dest_project,
+            dest.get_xnat_experiment_id()
+        )
+    except datman.xnat.XnatException:
+        logger.info(f"XNAT experiment {dest.get_xnat_experiment_id()}"
+                    " already exists. No changes made.")
+    except requests.HTTPError as e:
+        logger.error(f"Failed while sharing source ID {source} into dest ID "
+                     f"{dest} on XNAT. Reason - {e}")
 
 
 def share_redcap_record(session, shared_record):
@@ -200,12 +297,22 @@ def share_redcap_record(session, shared_record):
 
 
 class Record(object):
-    def __init__(self, record_dict, id_map=None):
-        self.record_id = record_dict['record_id']
-        self.id = self.__get_datman_id(record_dict['par_id'], id_map)
+    def __init__(self, record_dict, id_map=None, record_id_field='record_id',
+                 id_field='par_id', shared_id_prefix_field='shared_parid'):
+        try:
+            self.record_id = record_dict[record_id_field]
+        except KeyError:
+            # Try default in case user misconfigured their fields!
+            self.record_id = record_dict['record_id']
+        try:
+            subid = record_dict[id_field]
+        except KeyError:
+            # Try default in case user misconfigured their fields!
+            subid = record_dict['par_id']
+        self.id = self.__get_datman_id(subid, id_map)
         self.study = self.__get_study()
-        self.comment = record_dict['cmts']
-        self.shared_ids = self.__get_shared_ids(record_dict, id_map)
+        self.shared_ids = self.__get_shared_ids(record_dict, id_map,
+                                                shared_id_prefix_field)
 
     def matches_study(self, study_tag):
         if study_tag == self.study:
@@ -217,11 +324,11 @@ def __get_study(self):
             return None
         return self.id.study
 
-    def __get_shared_ids(self, record_dict, id_map):
+    def __get_shared_ids(self, record_dict, id_map, shared_id_prefix_field):
         keys = list(record_dict)
         shared_id_fields = []
         for key in keys:
-            if 'shared_parid' in key:
+            if shared_id_prefix_field in key:
                 shared_id_fields.append(key)
 
         shared_ids = []
@@ -234,7 +341,7 @@ def __get_shared_ids(self, record_dict, id_map):
             if subject_id is None:
                 # Badly named shared id value. Skip it.
                 continue
-            shared_ids.append(str(subject_id))
+            shared_ids.append(subject_id)
         return shared_ids
 
     def __get_datman_id(self, subid, id_map):

datman/xnat.py

@@ -981,6 +981,82 @@ def rename_experiment(self, project, subject, old_name, new_name):
             else:
                 raise e
 
+    def share_subject(self, source_project, source_sub, dest_project,
+                      dest_sub):
+        """Share an xnat subject into another project.
+
+        Args:
+            source_project (:obj:`str`): The name of the original project
+                the subject was uploaded to.
+            source_sub (:obj:`str`): The original ID of the subject to be
+                shared.
+            dest_project (:obj:`str`): The new project to add the subject to.
+            dest_sub (:obj:`str`): The ID to give the subject in the
+                destination project.
+
+        Raises:
+            XnatException: If the destination subject ID is already in use
+                or the source subject doesn't exist.
+            requests.HTTPError: If any unexpected behavior is experienced
+                while interacting with XNAT's API
+        """
+        # Ensure source subject exists, raises an exception if not
+        self.get_subject(source_project, source_sub)
+
+        url = (f"{self.server}/data/projects/{source_project}/subjects/"
+               f"{source_sub}/projects/{dest_project}?label={dest_sub}")
+
+        try:
+            self._make_xnat_put(url)
+        except requests.HTTPError as e:
+            if e.response.status_code == 409:
+                raise XnatException(
+                    f"Can't share {source_sub} as {dest_sub}, subject "
+                    "ID already exists.")
+            else:
+                raise e
+
+    def share_experiment(self, source_project, source_sub, source_exp,
+                         dest_project, dest_exp):
+        """Share an experiment into a new xnat project.
+
+        Note: The subject the experiment belongs to must have already been
+        shared to the destination project for experiment sharing to work.
+
+        Args:
+            source_project (:obj:`str`): The original project the experiment
+                belongs to.
+            source_sub (:obj:`str`): The original subject ID in the source
+                project.
+            source_exp (:obj:`str`): The original experiment name in the
+                source project.
+            dest_project (:obj:`str`): The project the experiment is to be
+                added to.
+            dest_exp (:obj:`str`): The name to apply to the experiment when
+                it is added to the destination project.
+
+        Raises:
+            XnatException: If the destination experiment ID is already in
+                use or the source experiment ID doesnt exist.
+            requests.HTTPError: If any unexpected behavior is experienced
+                while interacting with XNAT's API.
+        """
+        # Ensure source experiment exists, raises an exception if not
+        self.get_experiment(source_project, source_sub, source_exp)
+
+        url = (f"{self.server}/data/projects/{source_project}/subjects/"
+               f"{source_sub}/experiments/{source_exp}/projects/"
+               f"{dest_project}?label={dest_exp}")
+
+        try:
+            self._make_xnat_put(url)
+        except requests.HTTPError as e:
+            if e.response.status_code == 409:
+                raise XnatException(f"Can't share {source_exp} as {dest_exp}"
+                                    " experiment ID already exists")
+            else:
+                raise e
+
     def dismiss_autorun(self, experiment):
         """Mark the AutoRun.xml pipeline as finished.