Ditch (unneeded) coining to "derivatives"

bidscoin/bids.py

@@ -920,7 +920,7 @@ def __init__(self, yamlfile: Path, folder: Path=templatefolder, plugins: Iterabl
         subprefix = options['subprefix'] = options['subprefix'] or ''
         sesprefix = options['sesprefix'] = options['sesprefix'] or ''
 
-        # Append the existing .bidsignore data from the bidsfolder and make sure bidsignore, unknowntypes, ignoretypes and notderivative are lists
+        # Append the existing .bidsignore data from the bidsfolder and make sure bidsignore, unknowntypes and ignoretypes are lists
         if isinstance(options.get('bidsignore'), str):
             options['bidsignore'] = options['bidsignore'].split(';')
         bidsignorefile = folder.parents[1]/'.bidsignore'
@@ -929,7 +929,6 @@ def __init__(self, yamlfile: Path, folder: Path=templatefolder, plugins: Iterabl
         options['bidsignore']     = sorted(set(options.get('bidsignore'))) or []
         options['unknowntypes']   = options.get('unknowntypes')  or []
         options['ignoretypes']    = options.get('ignoretypes')   or []
-        options['notderivative']  = options.get('notderivative') or []
 
         # Make sure we get a proper plugin options and dataformat sections (use plugin default bidsmappings when a template bidsmap is loaded)
         if plugins:
@@ -2180,22 +2179,6 @@ def match_runvalue(attribute, pattern) -> bool:
     return match is not None
 
 
-def get_derivatives(datatype: Union[str, DataType], exceptions: Iterable=()) -> list:
-    """
-    Retrieves a list of suffixes that are stored in the derivatives folder (e.g. the qMRI maps). TODO: Replace with a more systematic/documented method
-    """
-
-    datatype = str(datatype)
-    if datatype == 'anat':
-        return [suffix for suffix in filerules[datatype].parametric.suffixes
-                if suffix not in tuple(exceptions) + ('UNIT1',)]        # The qMRI data (maps)
-    elif datatype == 'fmap':
-        return [suffix for typegroup in filerules[datatype] for suffix in filerules[datatype][typegroup].suffixes
-                if suffix not in exceptions and typegroup not in ('fieldmaps','pepolar')]    # The non-standard fmaps (file collections)
-    else:
-        return []
-
-
 def get_bidsvalue(bidsfile: Union[str, Path], bidskey: str, newvalue: str='') -> Union[Path, str]:
     """
     Sets the bidslabel, i.e. '*_bidskey-*_' is replaced with '*_bidskey-bidsvalue_'. If the key exists but is not in the

bidscoin/bidseditor.py

@@ -544,11 +544,7 @@ def update_subses_samples(self, dataformat: str):
                 subid, sesid = runitem.datasource.subid_sesid(subid, sesid or '')
                 bidsname     = runitem.bidsname(subid, sesid, not bids.check_ignore(datatype,self.bidsignore) and dtype not in self.ignoredatatypes)
                 ignore       = bids.check_ignore(datatype, self.bidsignore) or bids.check_ignore(bidsname+'.json', self.bidsignore, 'file')
-                exceptions   = self.output_bidsmap.options['notderivative']
-                if runitem.datasource.dynamicvalue(runitem.bids['suffix'], True, True) in bids.get_derivatives(dtype, exceptions):
-                    session  = self.bidsfolder/'derivatives'/'[manufacturer]'/subid/sesid
-                else:
-                    session  = self.bidsfolder/subid/sesid
+                session      = self.bidsfolder/subid/sesid
                 row_index    = self.ordered_file_index[dataformat][provenance]
 
                 samples_table.setItem(idx, 0, MyWidgetItem(f"{row_index+1:03d}", iseditable=False))
@@ -1917,14 +1913,10 @@ def get_suffixhelp(suffix: str, datatype: Union[str, DataType]) -> str:
     if not suffix:
         return "Please provide a suffix"
 
-    isderivative = ''
-    if suffix in bids.get_derivatives(datatype):
-        isderivative = '\nNB: This is a BIDS derivatives datatype'
-
     # Return the description for the suffix or a default text
     suffixes = bids.bidsschema.objects.suffixes
     if suffix in suffixes:
-        return f"{suffixes[suffix].display_name}\n{suffixes[suffix].description}{isderivative}"
+        return f"{suffixes[suffix].display_name}\n{suffixes[suffix].description}"
 
     return f"{suffix}\nAn unknown/private suffix"
 

bidscoin/heuristics/bidsmap_bids2bids.yaml

@@ -29,7 +29,6 @@ Options:
     bidsignore: [extra_data/, sub-*_ct.*]    # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
     unknowntypes: [extra_data]      # A list of datatypes that are converted to BIDS-like datatype folders
     ignoretypes: [exclude]          # A list of datatypes that are excluded / not converted to BIDS
-    notderivative: []               # A list of suffixes that should not be considered as a derivative datatype
     unzip:                          # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz'
   plugins:                          # List of plugins with plugin-specific key-value pairs (that can be used by the plugin)
     nibabel2bids:

bidscoin/heuristics/bidsmap_dccn.yaml

@@ -30,7 +30,6 @@ Options:
     bidsignore: [extra_data/, sub-*_ct.*]    # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
     unknowntypes: [extra_data]      # A list of datatypes that are converted to BIDS-like datatype folders
     ignoretypes: [exclude]          # A list of datatypes that are excluded / not converted to BIDS
-    notderivative: []               # A list of suffixes that should not be considered as a derivative datatype
     unzip:                          # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz'
   plugins:                          # List of plugins with plugin-specific key-value pairs (that can be used by the plugin)
     dcm2niix2bids:                  # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info

bidscoin/heuristics/bidsmap_sst.yaml

@@ -29,7 +29,6 @@ Options:
     bidsignore: [extra_data/, sub-*_ct.*]    # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
     unknowntypes: [extra_data]      # A list of datatypes that are converted to BIDS-like datatype folders
     ignoretypes: [exclude]          # A list of datatypes that are excluded / not converted to BIDS
-    notderivative: []               # A list of suffixes that should not be considered as a derivative datatype
     unzip:                          # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz'
   plugins:                          # List of plugins with plugin-specific key-value pairs (that can be used by the plugin)
     dcm2niix2bids:                  # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info

bidscoin/heuristics/schema.json

@@ -22,7 +22,6 @@
                   "sesprefix": { "type": ["string", "null"] },
                   "unknowntypes": { "type": ["array"] },
                   "ignoretypes": { "type": ["array"] },
-                  "notderivative": { "type": ["array"] },
                   "unzip": { "type": ["string", "null"] }
                },
                "additionalProperties": false

bidscoin/plugins/dcm2niix2bids.py

@@ -210,7 +210,6 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[N
 
     # Get started and see what dataformat we have
     options          = Plugin(bidsmap.plugins['dcm2niix2bids'])
-    exceptions: list = bidsmap.options.get('notderivative', [])
     bidsignore: list = bidsmap.options['bidsignore']
     fallback         = 'fallback' if options.get('fallback','y').lower() in ('y', 'yes', 'true') else ''
     datasource       = bids.get_datasource(session, Plugins({'dcm2niix2bids': options}))
@@ -268,11 +267,8 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[N
         LOGGER.info(f"--> Coining: {run.datasource}")
 
         # Create the BIDS session/datatype output folder
-        suffix = run.datasource.dynamicvalue(run.bids['suffix'], True, True)
-        if suffix in bids.get_derivatives(run.datatype, exceptions):
-            outfolder = bidsfolder/'derivatives'/manufacturer.replace(' ','')/subid/sesid/run.datatype
-        else:
-            outfolder = bidsses/run.datatype
+        suffix    = run.datasource.dynamicvalue(run.bids['suffix'], True, True)
+        outfolder = bidsses/run.datatype
         outfolder.mkdir(parents=True, exist_ok=True)
 
         # Compose the BIDS filename using the matched run
@@ -455,13 +451,9 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[N
         # Write out provenance data
         bids.bidsprov(bidsses, source, run, targets)
 
-        # Loop over all non-derivative targets (i.e. the produced output files) and edit the json sidecar data
+        # Loop over all targets (i.e. the produced output files) and edit the json sidecar data
         for target in sorted(targets):
 
-            # Editing derivative data is out of our scope
-            if target.relative_to(bidsfolder).parts[0] == 'derivatives':
-                continue
-
             # Load/copy over the source meta-data
             jsonfile = target.with_suffix('').with_suffix('.json')
             if not jsonfile.is_file():

bidscoin/plugins/nibabel2bids.py

@@ -234,9 +234,8 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> None:
                 json.dump(metadata, json_fid, indent=4)
 
         # Add an entry to the scans_table (we typically don't have useful data to put there)
-        if 'derivatives' not in bidsses.parts:
-            acq_time = dateutil.parser.parse(f"1925-01-01T{metadata.get('AcquisitionTime', '')}")
-            scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time.isoformat()
+        acq_time = dateutil.parser.parse(f"1925-01-01T{metadata.get('AcquisitionTime', '')}")
+        scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time.isoformat()
 
     if not runid:
         LOGGER.info(f"--> No {__name__} sourcedata found in: {session}")

docs/CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ## [dev]
 
+### Changed
+- Drop saving data in the derivatives folder (i.e. this was not required by BIDS after all)
+
 ## [4.4.0] - 2024-10-02
 
 ### Added

docs/options.rst

@@ -23,7 +23,6 @@ These setting can be used by all the BIDScoin tools:
 - ``datatypes``: Datatypes that are converted to BIDS. This can be useful for ignoring/excluding specific data types (without changing their mappings)
 - ``unknowntypes``: Datatypes that are not part of BIDS but that are converted to a BIDS-like entries in the BIDS folder
 - ``ignoretypes``: Datatypes that are excluded/not converted"""
-- ``notderivative``: A list of suffixes that should not be considered as a derivative data type (i.e. use this to make exceptions for what goes into ``bids/derivatives``)
 - ``unzip``: Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data. Use for instance '\*.tar.gz' if your source data looks like ``sub-01\01_MPRAGE\dcmfiles.tar.gz``, etc
 
 The core working of BIDScoin and its plugins can be tested by clicking the corresponding [Test] button and inspection of the terminal output.

tests/test_data/bidsmap.yaml

@@ -28,7 +28,6 @@ Options:
     sesprefix: ses-                 # The session prefix of the source data
     unknowntypes: [mrs, extra_data] # A list of datatypes that are converted to BIDS-like datatype folders
     ignoretypes: [exclude]          # A list of datatypes that are excluded/not converted to BIDS
-    notderivative: []               # A list of suffixes that should not be considered as a derivative datatype
     unzip:                          # Wildcard pattern to select tarballed/zip-files in the sourcefolders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz'
   plugins:                          # List of plugins with plugin-specific key-value pairs (that can be used by the plugin)
     dcm2niix2bids:                  # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info