Merge branch 'release/1.7.0'

CHANGELOG

@@ -1,8 +1,20 @@
 Changelog
 =========
+## [1.7.0] - 2021-01-15
+### Summary
+This release incorporates the telephone followup packet 3.2 module which
+includes the C2T form. Centers will be able to process data from forms like
+the T1 and C2T, along with the expanded telehealth regimen in the other
+forms provided by NACC due to the COVID-19 pandemic.
+
+### Added
+ * Unit tests for TFP module functionality
+ * Re-add skipping logic to TFP form a3 from the updated DED for TFP
+ * Added the tele_ prefix to drug id filter for form A4
+
 ## [1.6.0] - 2020-12-15
 ### Summary
-This change was implemented after getting feedback from OHSU. Some ADRCs do 
+This change was implemented after getting feedback from OHSU. Some ADRCs do
 not have the optional forms like A2 or A3 in their REDCap project at all, since
 they will not be used for that center. NACCulator used to run with the
 requirement that all forms be present in a REDCap project, whether they were

README.md

@@ -57,7 +57,8 @@ the `-file` flag._
       -h, --help            show this help message and exit
       -fvp                  Set this flag to process as FVP data
       -ivp                  Set this flag to process as IVP data
-      -tfp                  Set this flag to process as Telephone Followup Packet data
+      -tfp                  Set this flag to process as Telephone Followup Packet v3.2 data
+      -tfp3                 Set this flag to process as TFP v3.0 (pre-2020) data
       -np                   Set this flag to process as Neuropathology data
       -m                    Set this flag to process as Milestone data
       -csf                  Set this flag to process as NACC BIDSS CSF data

nacc/ftld/fvp/builder.py

@@ -472,6 +472,8 @@ def update_header(record, packet):
     for header in packet:
         header.PACKET = "FF"
         header.FORMID = header.form_name
+        if header.FORMID == "Z1X":
+            header.PACKET = "F"
         header.FORMVER = 3
         header.ADCID = record['adcid']
         header.PTID = record['ptid']

nacc/ftld/ivp/builder.py

@@ -473,6 +473,8 @@ def update_header(record, packet):
     for header in packet:
         header.PACKET = "IF"
         header.FORMID = header.form_name
+        if header.FORMID == "Z1X":
+            header.PACKET = "I"
         header.FORMVER = 3
         header.ADCID = record['adcid']
         header.PTID = record['ptid']

nacc/redcap2nacc.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 ###############################################################################
-# Copyright 2015-2020 University of Florida. All rights reserved.
+# Copyright 2015-2021 University of Florida. All rights reserved.
 # This file is part of UF CTS-IT's NACCulator project.
 # Use of this source code is governed by the license found in the LICENSE file.
 ###############################################################################
@@ -21,6 +21,7 @@
 from nacc.uds3.np import builder as np_builder
 from nacc.uds3.fvp import builder as fvp_builder
 from nacc.uds3.tfp import builder as tfp_builder
+from nacc.uds3.tfp.v3_2 import builder as tfp_new_builder
 from nacc.uds3.m import builder as m_builder
 from nacc.lbd.ivp import builder as lbd_ivp_builder
 from nacc.lbd.fvp import builder as lbd_fvp_builder
@@ -216,7 +217,26 @@ def check_redcap_event(options, record) -> bool:
     elif options.np:
         event_name = 'neuropath'
     elif options.tfp:
-        event_name = 'telephone'
+        event_name = 'follow'
+        try:
+            followup_match = record['tvp_z1x_checklist_complete']
+            if followup_match in ['', '0']:
+                return False
+        except KeyError:
+            try:
+                followup_match = record['tfp_z1x_complete']
+                if followup_match in ['', '0']:
+                    return False
+            except KeyError:
+                try:
+                    followup_match = record['tele_z1x_complete']
+                    if followup_match in ['', '0']:
+                        return False
+                except KeyError:
+                    print("Could not find a REDCap field for TFP Z1X form.")
+                    return False
+    elif options.tfp3:
+        event_name = 'tele'
     elif options.m:
         event_name = 'milestone'
 
@@ -284,44 +304,66 @@ def set_to_zero_if_blank(*field_names):
                 field.value = 0
 
     # B8 2.
-    if packet['PARKSIGN'] == 1:
-        set_to_zero_if_blank(
-            'RESTTRL', 'RESTTRR', 'SLOWINGL', 'SLOWINGR', 'RIGIDL', 'RIGIDR',
-            'BRADY', 'PARKGAIT', 'POSTINST')
+    try:
+        if packet['PARKSIGN'] == 1:
+            set_to_zero_if_blank(
+                'RESTTRL', 'RESTTRR', 'SLOWINGL', 'SLOWINGR', 'RIGIDL', 'RIGIDR',
+                'BRADY', 'PARKGAIT', 'POSTINST')
+    except KeyError:
+        pass
 
     # B8 3.
-    if packet['CVDSIGNS'] == 1:
-        set_to_zero_if_blank('CORTDEF', 'SIVDFIND', 'CVDMOTL', 'CVDMOTR',
-                             'CORTVISL', 'CORTVISR', 'SOMATL', 'SOMATR')
+    try:
+        if packet['CVDSIGNS'] == 1:
+            set_to_zero_if_blank('CORTDEF', 'SIVDFIND', 'CVDMOTL', 'CVDMOTR',
+                                'CORTVISL', 'CORTVISR', 'SOMATL', 'SOMATR')
+    except KeyError:
+        pass
 
     # B8 5.
-    if packet['PSPCBS'] == 1:
-        set_to_zero_if_blank(
-            'PSPCBS', 'EYEPSP', 'DYSPSP', 'AXIALPSP', 'GAITPSP', 'APRAXSP',
-            'APRAXL', 'APRAXR', 'CORTSENL', 'CORTSENR', 'ATAXL', 'ATAXR',
-            'ALIENLML', 'ALIENLMR', 'DYSTONL', 'DYSTONR', 'MYOCLLT', 'MYOCLRT')
+    try:
+        if packet['PSPCBS'] == 1:
+            set_to_zero_if_blank(
+                'PSPCBS', 'EYEPSP', 'DYSPSP', 'AXIALPSP', 'GAITPSP', 'APRAXSP',
+                'APRAXL', 'APRAXR', 'CORTSENL', 'CORTSENR', 'ATAXL', 'ATAXR',
+                'ALIENLML', 'ALIENLMR', 'DYSTONL', 'DYSTONR', 'MYOCLLT',
+                'MYOCLRT')
+    except KeyError:
+        pass
 
     # D1 4.
-    if packet['DEMENTED'] == 1:
-        set_to_zero_if_blank(
-                'AMNDEM', 'PCA', 'PPASYN', 'FTDSYN', 'LBDSYN', 'NAMNDEM')
+    try:
+        if packet['DEMENTED'] == 1:
+            set_to_zero_if_blank(
+                    'AMNDEM', 'PCA', 'PPASYN', 'FTDSYN', 'LBDSYN', 'NAMNDEM')
+    except KeyError:
+        pass
 
     # D1 5.
-    if packet['DEMENTED'] == 0:
-        set_to_zero_if_blank(
-                'MCIAMEM', 'MCIAPLUS', 'MCINON1', 'MCINON2', 'IMPNOMCI')
+    try:
+        if packet['DEMENTED'] == 0:
+            set_to_zero_if_blank(
+                    'MCIAMEM', 'MCIAPLUS', 'MCINON1', 'MCINON2', 'IMPNOMCI')
+    except KeyError:
+        pass
 
     # D1 11-39.
-    set_to_zero_if_blank(
-        'ALZDIS', 'LBDIS', 'MSA', 'PSP', 'CORT', 'FTLDMO', 'FTLDNOS', 'CVD',
-        'ESSTREM', 'DOWNS', 'HUNT', 'PRION', 'BRNINJ', 'HYCEPH', 'EPILEP',
-        'NEOP', 'HIV', 'OTHCOG', 'DEP', 'BIPOLDX', 'SCHIZOP', 'ANXIET',
-        'DELIR', 'PTSDDX', 'OTHPSY', 'ALCDEM', 'IMPSUB', 'DYSILL', 'MEDS',
-        'COGOTH', 'COGOTH2', 'COGOTH3')
+    try:
+        set_to_zero_if_blank(
+            'ALZDIS', 'LBDIS', 'MSA', 'PSP', 'CORT', 'FTLDMO', 'FTLDNOS', 'CVD',
+            'ESSTREM', 'DOWNS', 'HUNT', 'PRION', 'BRNINJ', 'HYCEPH', 'EPILEP',
+            'NEOP', 'HIV', 'OTHCOG', 'DEP', 'BIPOLDX', 'SCHIZOP', 'ANXIET',
+            'DELIR', 'PTSDDX', 'OTHPSY', 'ALCDEM', 'IMPSUB', 'DYSILL', 'MEDS',
+            'COGOTH', 'COGOTH2', 'COGOTH3')
+    except KeyError:
+        pass
 
     # D2 11.
-    if packet['ARTH'] == 1:
-        set_to_zero_if_blank('ARTUPEX', 'ARTLOEX', 'ARTSPIN', 'ARTUNKN')
+    try:
+        if packet['ARTH'] == 1:
+            set_to_zero_if_blank('ARTUPEX', 'ARTLOEX', 'ARTSPIN', 'ARTUNKN')
+    except KeyError:
+        pass
 
 
 def convert(fp, options, out=sys.stdout, err=sys.stderr):
@@ -358,6 +400,8 @@ def convert(fp, options, out=sys.stdout, err=sys.stderr):
             elif options.fvp:
                 packet = fvp_builder.build_uds3_fvp_form(record)
             elif options.tfp:
+                packet = tfp_new_builder.build_uds3_tfp_new_form(record)
+            elif options.tfp3:
                 packet = tfp_builder.build_uds3_tfp_form(record)
             elif options.m:
                 packet = m_builder.build_uds3_m_form(record)
@@ -369,12 +413,11 @@ def convert(fp, options, out=sys.stdout, err=sys.stderr):
             traceback.print_exc()
             continue
 
-        if not options.np and not options.m and not options.tfp and not \
-            options.lbd and not options.lbdsv and not options.ftld and not \
-            options.csf:
+        if not (options.np or options.m or options.lbd or options.lbdsv or
+                options.ftld or options.csf):
             set_blanks_to_zero(packet)
 
-        if options.m:
+        if options.m or options.tfp:
             blanks_uds3.set_zeros_to_blanks(packet)
 
         warnings = []
@@ -439,7 +482,10 @@ def parse_args(args=None):
         help='Set this flag to process as ivp data')
     option_group.add_argument(
         '-tfp', action='store_true', dest='tfp',
-        help='Set this flag to process as tfp data')
+        help='Set this flag to process as tfp version 3.2 data')
+    option_group.add_argument(
+        '-tfp3', action='store_true', dest='tfp3',
+        help='Set this flag to process as tfp version 3.0 (pre-June 2020) data')
     option_group.add_argument(
         '-np', action='store_true', dest='np',
         help='Set this flag to process as np data')
@@ -483,8 +529,8 @@ def parse_args(args=None):
     options = parser.parse_args(args)
     # Defaults to processing of ivp.
     # TODO this can be changed in future to process fvp by default.
-    if not (options.ivp or options.fvp or options.tfp or options.np or
-            options.m or options.csf or options.filter):
+    if not (options.ivp or options.fvp or options.tfp or options.tfp3 or 
+            options.np or options.m or options.csf or options.filter):
         options.ivp = True
 
     return options

nacc/uds3/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright 2015-2020 University of Florida. All rights reserved.
+# Copyright 2015-2021 University of Florida. All rights reserved.
 # This file is part of UF CTS-IT's NACCulator project.
 # Use of this source code is governed by the license found in the LICENSE file.
 ###############################################################################

nacc/uds3/blanks.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright 2015-2016 University of Florida. All rights reserved.
+# Copyright 2015-2021 University of Florida. All rights reserved.
 # This file is part of UF CTS-IT's NACCulator project.
 # Use of this source code is governed by the license found in the LICENSE file.
 ###############################################################################
@@ -53,13 +53,15 @@ def convert_rule_to_python(name: str, rule: str) -> bool:
         'NPPATH9': _blanking_rule_dummy,
         'NPPATH10': _blanking_rule_dummy,
         'NPPATH11': _blanking_rule_dummy,
+        # TFP 3.2 skip rules
+        'TELMILE': _blanking_rule_telmile,
     }
 
     single_value = re.compile(
-        r"Blank if( Question(s?))? *\w+ (?P<key>\w+) *(?P<eq>=|ne)"
+        r"Blank if( Question(s?))? *\w+\.? (?P<key>\w+) *(?P<eq>=|ne)"
         r" (?P<value>\d+)([^-]|$)")
     range_values = re.compile(
-        r"Blank if( Question(s?))? *\w+ (?P<key>\w+) *(?P<eq>=|ne)"
+        r"Blank if( Question(s?))? *\w+\.? (?P<key>\w+) *(?P<eq>=|ne)"
         r" (?P<start>\d+)-(?P<stop>\d+)( |$)")
 
     # First, check to see if the rule is a "Special Case"
@@ -140,6 +142,13 @@ def _blanking_rule_learned():
     return lambda packet: packet['REFERSC'] in (3, 4, 5, 6, 8, 9)
 
 
+def _blanking_rule_telmile():
+    # 'Blank if Question 3 TELINPER = 1 (Yes)'
+    # 'Blank if Question 3 TELINPER = 9 (Unknown)'
+    # 'Blank if this is the first telephone packet submitted for the subject.'
+    return lambda packet: packet['TELINPER'] in (1, 9)
+
+
 def set_zeros_to_blanks(packet):
     """ Sets specific fields to zero if they meet certain criteria """
     def set_to_blank_if_zero(*field_names):
@@ -148,18 +157,29 @@ def set_to_blank_if_zero(*field_names):
             if field == 0:
                 field.value = ''
     # M1
-    if packet['DECEASED'] == 1 or packet['DISCONT'] == 1:
-        set_to_blank_if_zero(
-            'RENURSE', 'RENAVAIL', 'RECOGIM', 'REJOIN', 'REPHYILL',
-            'REREFUSE', 'FTLDDISC', 'CHANGEMO', 'CHANGEDY', 'CHANGEYR',
-            'PROTOCOL', 'ACONSENT', 'RECOGIM', 'REPHYILL', 'NURSEMO',
-            'NURSEDY', 'NURSEYR', 'FTLDREAS', 'FTLDREAX')
-    elif packet['DECEASED'] == 1:
-        # for just dead
-        set_to_blank_if_zero('DISCONT')
-    elif packet['DISCONT'] == 1:
-        # for just discont
-        set_to_blank_if_zero('DECEASED')
+    try:
+        if packet['DECEASED'] == 1 or packet['DISCONT'] == 1:
+            set_to_blank_if_zero(
+                'RENURSE', 'RENAVAIL', 'RECOGIM', 'REJOIN', 'REPHYILL',
+                'REREFUSE', 'FTLDDISC', 'CHANGEMO', 'CHANGEDY', 'CHANGEYR',
+                'PROTOCOL', 'ACONSENT', 'RECOGIM', 'REPHYILL', 'NURSEMO',
+                'NURSEDY', 'NURSEYR', 'FTLDREAS', 'FTLDREAX')
+        elif packet['DECEASED'] == 1:
+            # for just dead
+            set_to_blank_if_zero('DISCONT')
+        elif packet['DISCONT'] == 1:
+            # for just discont
+            set_to_blank_if_zero('DECEASED')
+    except KeyError:
+        pass
+    # TFP
+    try:
+        if packet['RESPVAL'] == 1:
+            set_to_blank_if_zero(
+                'RESPHEAR', 'RESPDIST', 'RESPINTR', 'RESPDISN', 'RESPFATG',
+                'RESPEMOT', 'RESPASST', 'RESPOTH')
+    except KeyError:
+        pass
 
 
 def main():

nacc/uds3/filters.py

@@ -140,7 +140,7 @@ def filter_replace_drug_id_do(input_ptr, output_ptr):
     write_headers(reader, output)
     for record in reader:
         count = 0
-        prefixes = ['', 'fu_']
+        prefixes = ['', 'fu_', 'tele_']
         for prefix in prefixes:
             for i in range(1, 31):
                 col_name = prefix + 'drugid_' + str(i)