clean up

garjus/automations/__init__.py

@@ -290,20 +290,35 @@ def _run_etl_nihexaminer(project):
 
         # Check for blanks
         has_blank = False
-        check_fields = [flank_field, nback_field, shift_field, cpt_field,
-        'dot_count_tot',
-        'anti_trial_1',
-        'anti_trial_2',
-        'correct_f',
-        'correct_l',
-        'correct_animal',
-        'correct_veg',
-        'repetition_f',
-        'rule_vio_f',
-        'repetition_l',
-        'rule_vio_l', 'repetition_animal', 'rule_vio_animal', 'repetition_veg',
-        'rule_vio_veg', 'brs_1', 'brs_2', 'brs_3', 'brs_4',
-        'brs_5', 'brs_6', 'brs_7', 'brs_8', 'brs_9']
+        check_fields = [
+            flank_field,
+            nback_field,
+            shift_field,
+            cpt_field,
+            'dot_count_tot',
+            'anti_trial_1',
+            'anti_trial_2',
+            'correct_f',
+            'correct_l',
+            'correct_animal',
+            'correct_veg',
+            'repetition_f',
+            'rule_vio_f',
+            'repetition_l',
+            'rule_vio_l',
+            'repetition_animal',
+            'rule_vio_animal',
+            'repetition_veg',
+            'rule_vio_veg',
+            'brs_1',
+            'brs_2',
+            'brs_3',
+            'brs_4',
+            'brs_5',
+            'brs_6',
+            'brs_7',
+            'brs_8',
+            'brs_9']
 
         for k in check_fields:
             if r[k] == '' and k != done_field:
@@ -375,7 +390,6 @@ def _run_etl_nihexaminer(project):
                 'cf2_rv': int(r['rule_vio_cloth'])
             })
 
-
         with tempfile.TemporaryDirectory() as tmpdir:
             # Get files needed
             flank_file = f'{tmpdir}/flanker.csv'
@@ -499,9 +513,6 @@ def _run_scan_automations(automations, garjus, project):
                 sess_field,
                 sess_suffix)
 
-            # TODO: build the scan table and append each event, then run
-            # autoarchive once
-
             # Run
             logger.debug(f'running xnat_auto_archive:{project}:{events}')
             results += xnat_auto_archive.process_project(
@@ -542,7 +553,8 @@ def _make_scan_table(
     events,
     date_field,
     sess_field,
-    scan_suffix):
+    scan_suffix,
+):
     """Make the scan table, linking source to destination subject/session."""
     data = []
     id2subj = {}
@@ -577,7 +589,7 @@ def _make_scan_table(
     for r in rec:
         d = {}
         try:
-           d['dst_subject'] = id2subj.get(r[def_field])
+            d['dst_subject'] = id2subj.get(r[def_field])
         except KeyError:
             logger.warn(f'blank subject number:{r[def_field]}')
             continue
@@ -619,7 +631,7 @@ def _session_relabels(scan_data, site_data):
 def _load(project, record_id, event_id, data):
     data[project.def_field] = record_id
     data['redcap_event_name'] = event_id
-    data = {k: str(v) for k,v in data.items()}
+    data = {k: str(v) for k, v in data.items()}
 
     try:
         response = project.import_records([data])

garjus/automations/edat_convert2tab.py

@@ -5,7 +5,7 @@
 from ..utils_redcap import upload_file, download_file
 
 
-logger = logging.getLogger('garjus.automations.edat_limbo2redcap')
+logger = logging.getLogger('garjus.automations.edat_convert2tab')
 
 
 def process_project(

garjus/automations/edat_limbo2redcap.py

@@ -88,7 +88,7 @@ def process_project(
         try:
             result = upload_file(
                 project,
-                record_id, 
+                record_id,
                 raw_field,
                 edat_file,
                 event_id=event)
@@ -103,13 +103,13 @@ def process_project(
 
         logger.info(f'uploaded:{subj}:{event}')
         results.append({
-                    'result': 'COMPLETE',
-                    'description': 'edat_limbo2redcap',
-                    'category': 'edat_limbo2redcap',
-                    'subject': subj,
-                    'session': '',
-                    'scan': '',
-                    'event': event,
-                    'field': raw_field})
+            'result': 'COMPLETE',
+            'description': 'edat_limbo2redcap',
+            'category': 'edat_limbo2redcap',
+            'subject': subj,
+            'session': '',
+            'scan': '',
+            'event': event,
+            'field': raw_field})
 
     return results

garjus/automations/edat_redcap2xnat.py

@@ -105,7 +105,7 @@ def process_project(
                 scan['SESSION'],
                 scan['SCANID'],
                 xnat_scan_resource)
-    
+
             # Upload file to xnat resource
             result = upload_file(tab_file, resource)
 

garjus/automations/etl_traitadjtask.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import numpy as np
 
-from ...utils_redcap import get_redcap, download_file, field2events
+from ...utils_redcap import field2events, download_file
 
 
 tab_field = ''
@@ -97,11 +97,11 @@ def etl_tat(project, record_id, event_id, tab_field):
     # Download the tab file from redcap to tmp
     tmpdir = tempfile.mkdtemp()
     basename = '{}-{}-{}.txt'.format(record_id, event_id, tab_field)
-    tab_file = f'tmpdir/basename'
-    result = utils_redcap.download_file(
+    tab_file = f'{tmpdir}/{basename}'
+    result = download_file(
         project, record_id, tab_field, tab_file, event_id=event_id)
     if not result:
-        logging.error('{}:{}:{}'.format(record_id, event_id, 'download failed'))
+        logging.error(f'download failed:{record_id}:{event_id}')
         return
 
     # Extract the data
@@ -172,7 +172,7 @@ def process_project(project):
             continue
 
         # Do the ETL
-        etl_tat(project, record_id, event_id) #, tab_field)
+        etl_tat(project, record_id, event)
 
         logging.debug(visit + ':uploaded')
         results.append({
@@ -181,6 +181,5 @@ def process_project(project):
             'subject': subj,
             'event': event,
             'field': tab_field})
-
 
     return results

garjus/automations/xnat_add_slicetiming.py

@@ -55,7 +55,7 @@ def process_project(
             # upload to no slicetiming
             new_res = res.parent().resource('JSON_MissingSliceTiming')
             if new_res.exists():
-                logging.debug(f'JSON_MissingSliceTiming already exists:{full_path}')
+                logging.debug(f'JSON_MissingSliceTiming exists:{full_path}')
             else:
                 logging.debug(f'saving to JSON_MissingSliceTiming:{full_path}')
                 new_res.file(src).put(dst)
@@ -86,5 +86,3 @@ def add_slicetiming(jsonfile, slicetiming):
 def has_slicetiming(jsonfile):
     with open(jsonfile, 'r') as f:
         return ('SliceTiming' in f.read())
-
-

garjus/automations/xnat_auto_archive.py

@@ -57,7 +57,7 @@ def process_project(
             dst_proj,
             dst_subj,
             dst_sess,
-            ))
+        ))
 
         garjus.copy_session(
             src_proj,

garjus/automations/xnat_ma3stats2voltxt.py

@@ -1,15 +1,8 @@
 """Downlad stats upload to vol_txt in XNAT."""
 import logging
-import subprocess as sb
 import tempfile
-import os
-import pathlib
-import glob
 import csv
 
-from .. import utils_dcm2nii
-from .. import utils_xnat
-
 
 logger = logging.getLogger('garjus.automations.xnat_ma3stats2voltxt')
 
@@ -85,9 +78,9 @@ def process_project(
 
 def stats2voltxt(stats, voltxt):
     with open(stats, newline='') as f:
-      reader = csv.reader(f)
-      keys = next(reader)
-      values = next(reader)
+        reader = csv.reader(f)
+        keys = next(reader)
+        values = next(reader)
 
     # Remove first two items that are not found in MA v2
     keys = keys[2:]

garjus/automations/xnat_relabel_scans.py

@@ -25,7 +25,7 @@ def relabel_scans(xnat, project, relabels):
 
     # get a list of scans from the project
     scan_uri = '{}&project={}'.format(SCAN_URI, project)
-    json_data = json.loads(xnat._exec(scan_uri, 'GET'),  strict=False)
+    json_data = json.loads(xnat._exec(scan_uri, 'GET'), strict=False)
     scan_list = json_data['ResultSet']['Result']
 
     # iterate scan and relabel if needed
@@ -46,11 +46,11 @@ def relabel_scans(xnat, project, relabels):
             new_type = relabels[scan_type]
             scan_obj.attrs.set('xnat:imagescandata/type', new_type)
             results.append({
-                    'description': f'xnat_relabel_scans:{new_type}',
-                    'result': 'COMPLETE',
-                    'category': 'xnat_relabel_scans',
-                    'subject': subj,
-                    'session': sess,
-                    'scan': scan})
+                'description': f'xnat_relabel_scans:{new_type}',
+                'result': 'COMPLETE',
+                'category': 'xnat_relabel_scans',
+                'subject': subj,
+                'session': sess,
+                'scan': scan})
 
     return results

garjus/automations/xnat_relabel_sessions.py

@@ -24,15 +24,16 @@
 
 def process_project(xnat, project, relabels, replace):
     """Apply relabels to project sessions."""
-    results = relabel_sessions(xnat, project, relabels, replace=replace, overwrite=True)
+    results = relabel_sessions(
+        xnat, project, relabels, replace=replace, overwrite=True)
+
     return results
 
 
 def relabel(xnat, session, relabels, overwrite=False, replace=[]):
     cur_sess = session['label']
     cur_proj = session['project']
     cur_subj = session['subject_label']
-    cur_site = session['xnat:imagesessiondata/acquisition_site']
     allowed_k1 = ['session_type', 'session_label']
     allowed_k2 = ['session_type', 'site']
     mset = {}
@@ -57,31 +58,28 @@ def relabel(xnat, session, relabels, overwrite=False, replace=[]):
 
         if (overwrite is False) or (session[k2] and (session[k2] not in replace)):
             # There's already a value there
-            #print('already set:{}:{}'.format(k2, session[k2]))
             continue
         elif session[k2] == v2:
             # have we already set this to the same thing
-            #print('already set:{}:{}'.format(k2, session[k2]))
             continue
 
         # Add the relabel to mset
         if k2 == 'site':
             # We have to use the full path for site, no shortcut
             mset['xnat:imagesessiondata/acquisition_site'] = v2
         else:
-            mset[k2] =  v2
+            mset[k2] = v2
 
     # Check for empty set
     if not mset:
-        #print('{}:nothing to set'.format(cur_sess))
         return None
 
     # Connect to the session on xnat and apply new values
     logging.info(f'{cur_proj}:{cur_sess}:setting:{mset}')
     sess_obj = xnat.select_session(cur_proj, cur_subj, cur_sess)
     sess_obj.attrs.mset(mset)
 
-    result =  {
+    result = {
         'description': 'xnat_relabel_session',
         'result': 'COMPLETE',
         'category': 'xnat_relabel_session',
@@ -96,7 +94,7 @@ def relabel_sessions(xnat, project, relabels, overwrite=False, replace=[]):
 
     # get a list of sessions from the project
     sess_uri = '{}&project={}'.format(SESS_URI, project)
-    json_data = json.loads(xnat._exec(sess_uri, 'GET'),  strict=False)
+    json_data = json.loads(xnat._exec(sess_uri, 'GET'), strict=False)
     sessions = json_data['ResultSet']['Result']
 
     # iterate and relabel as needed

garjus/cli.py

@@ -108,8 +108,16 @@ def d2q():
 
 @cli.command('update')
 @click.argument(
-    'choice', 
-    type=click.Choice(['stats' ,'issues', 'progress', 'automations', 'compare', 'tasks', 'analyses']),
+    'choice',
+    type=click.Choice([
+        'stats',
+        'issues',
+        'progress',
+        'automations',
+        'compare',
+        'tasks',
+        'analyses'
+    ]),
     required=False,
     nargs=-1)
 @click.option('--project', '-p', 'project', multiple=True)
@@ -185,8 +193,10 @@ def export_pdf(src, project):
 
 @cli.command('image03')
 @click.option('--project', '-p', 'project', required=True)
-@click.option('--start', '-s', 'startdate', type=click.DateTime(formats=['%Y-%m-%d']))
-@click.option('--end', '-e', 'enddate', type=click.DateTime(formats=['%Y-%m-%d']))
+@click.option(
+    '--start', '-s', 'startdate', type=click.DateTime(formats=['%Y-%m-%d']))
+@click.option(
+    '--end', '-e', 'enddate', type=click.DateTime(formats=['%Y-%m-%d']))
 def image03(project, startdate, enddate):
     click.echo('garjus! image03')
     g = Garjus()
@@ -222,8 +232,6 @@ def delete(project, proctype):
 
 @cli.command('dashboard')
 def dashboard():
-    import sys
-    import os
     import webbrowser
     url = 'http://localhost:8050'
 

garjus/compare/__init__.py

@@ -74,4 +74,3 @@ def make_double_report(proj_primary, proj_secondary, pdf_file, excel_file):
     p2 = proj_secondary.export_project_info().get('project_title')
     logger.debug(f'compare {p1} to {p2}')
     run_compare(proj_primary, proj_secondary, pdf_file, excel_file)
-