From 4ba5ff4b7400aa5208e166646fef78f88a1c2c1f Mon Sep 17 00:00:00 2001 From: Marcel Zwiers Date: Fri, 4 Oct 2024 13:01:50 +0200 Subject: [PATCH] Ditch (unneeded) coining to "derivatives" --- bidscoin/bids.py | 19 +------------------ bidscoin/bidseditor.py | 12 ++---------- bidscoin/heuristics/bidsmap_bids2bids.yaml | 1 - bidscoin/heuristics/bidsmap_dccn.yaml | 1 - bidscoin/heuristics/bidsmap_sst.yaml | 1 - bidscoin/heuristics/schema.json | 1 - bidscoin/plugins/dcm2niix2bids.py | 14 +++----------- bidscoin/plugins/nibabel2bids.py | 5 ++--- docs/CHANGELOG.md | 3 +++ docs/options.rst | 1 - tests/test_data/bidsmap.yaml | 1 - 11 files changed, 11 insertions(+), 48 deletions(-) diff --git a/bidscoin/bids.py b/bidscoin/bids.py index aa913e21..b7d71ea5 100644 --- a/bidscoin/bids.py +++ b/bidscoin/bids.py @@ -920,7 +920,7 @@ def __init__(self, yamlfile: Path, folder: Path=templatefolder, plugins: Iterabl subprefix = options['subprefix'] = options['subprefix'] or '' sesprefix = options['sesprefix'] = options['sesprefix'] or '' - # Append the existing .bidsignore data from the bidsfolder and make sure bidsignore, unknowntypes, ignoretypes and notderivative are lists + # Append the existing .bidsignore data from the bidsfolder and make sure bidsignore, unknowntypes and ignoretypes are lists if isinstance(options.get('bidsignore'), str): options['bidsignore'] = options['bidsignore'].split(';') bidsignorefile = folder.parents[1]/'.bidsignore' @@ -929,7 +929,6 @@ def __init__(self, yamlfile: Path, folder: Path=templatefolder, plugins: Iterabl options['bidsignore'] = sorted(set(options.get('bidsignore'))) or [] options['unknowntypes'] = options.get('unknowntypes') or [] options['ignoretypes'] = options.get('ignoretypes') or [] - options['notderivative'] = options.get('notderivative') or [] # Make sure we get a proper plugin options and dataformat sections (use plugin default bidsmappings when a template bidsmap is loaded) if plugins: @@ -2180,22 +2179,6 @@ def match_runvalue(attribute, pattern) -> bool: return match is not None -def get_derivatives(datatype: Union[str, DataType], exceptions: Iterable=()) -> list: - """ - Retrieves a list of suffixes that are stored in the derivatives folder (e.g. the qMRI maps). TODO: Replace with a more systematic/documented method - """ - - datatype = str(datatype) - if datatype == 'anat': - return [suffix for suffix in filerules[datatype].parametric.suffixes - if suffix not in tuple(exceptions) + ('UNIT1',)] # The qMRI data (maps) - elif datatype == 'fmap': - return [suffix for typegroup in filerules[datatype] for suffix in filerules[datatype][typegroup].suffixes - if suffix not in exceptions and typegroup not in ('fieldmaps','pepolar')] # The non-standard fmaps (file collections) - else: - return [] - - def get_bidsvalue(bidsfile: Union[str, Path], bidskey: str, newvalue: str='') -> Union[Path, str]: """ Sets the bidslabel, i.e. '*_bidskey-*_' is replaced with '*_bidskey-bidsvalue_'. If the key exists but is not in the diff --git a/bidscoin/bidseditor.py b/bidscoin/bidseditor.py index 58a401ac..736f4271 100755 --- a/bidscoin/bidseditor.py +++ b/bidscoin/bidseditor.py @@ -544,11 +544,7 @@ def update_subses_samples(self, dataformat: str): subid, sesid = runitem.datasource.subid_sesid(subid, sesid or '') bidsname = runitem.bidsname(subid, sesid, not bids.check_ignore(datatype,self.bidsignore) and dtype not in self.ignoredatatypes) ignore = bids.check_ignore(datatype, self.bidsignore) or bids.check_ignore(bidsname+'.json', self.bidsignore, 'file') - exceptions = self.output_bidsmap.options['notderivative'] - if runitem.datasource.dynamicvalue(runitem.bids['suffix'], True, True) in bids.get_derivatives(dtype, exceptions): - session = self.bidsfolder/'derivatives'/'[manufacturer]'/subid/sesid - else: - session = self.bidsfolder/subid/sesid + session = self.bidsfolder/subid/sesid row_index = self.ordered_file_index[dataformat][provenance] samples_table.setItem(idx, 0, MyWidgetItem(f"{row_index+1:03d}", iseditable=False)) @@ -1917,14 +1913,10 @@ def get_suffixhelp(suffix: str, datatype: Union[str, DataType]) -> str: if not suffix: return "Please provide a suffix" - isderivative = '' - if suffix in bids.get_derivatives(datatype): - isderivative = '\nNB: This is a BIDS derivatives datatype' - # Return the description for the suffix or a default text suffixes = bids.bidsschema.objects.suffixes if suffix in suffixes: - return f"{suffixes[suffix].display_name}\n{suffixes[suffix].description}{isderivative}" + return f"{suffixes[suffix].display_name}\n{suffixes[suffix].description}" return f"{suffix}\nAn unknown/private suffix" diff --git a/bidscoin/heuristics/bidsmap_bids2bids.yaml b/bidscoin/heuristics/bidsmap_bids2bids.yaml index d4c3febf..dfbf20a0 100644 --- a/bidscoin/heuristics/bidsmap_bids2bids.yaml +++ b/bidscoin/heuristics/bidsmap_bids2bids.yaml @@ -29,7 +29,6 @@ Options: bidsignore: [extra_data/, sub-*_ct.*] # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv] unknowntypes: [extra_data] # A list of datatypes that are converted to BIDS-like datatype folders ignoretypes: [exclude] # A list of datatypes that are excluded / not converted to BIDS - notderivative: [] # A list of suffixes that should not be considered as a derivative datatype unzip: # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz' plugins: # List of plugins with plugin-specific key-value pairs (that can be used by the plugin) nibabel2bids: diff --git a/bidscoin/heuristics/bidsmap_dccn.yaml b/bidscoin/heuristics/bidsmap_dccn.yaml index 97fafcd8..1d2384f8 100644 --- a/bidscoin/heuristics/bidsmap_dccn.yaml +++ b/bidscoin/heuristics/bidsmap_dccn.yaml @@ -30,7 +30,6 @@ Options: bidsignore: [extra_data/, sub-*_ct.*] # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv] unknowntypes: [extra_data] # A list of datatypes that are converted to BIDS-like datatype folders ignoretypes: [exclude] # A list of datatypes that are excluded / not converted to BIDS - notderivative: [] # A list of suffixes that should not be considered as a derivative datatype unzip: # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz' plugins: # List of plugins with plugin-specific key-value pairs (that can be used by the plugin) dcm2niix2bids: # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info diff --git a/bidscoin/heuristics/bidsmap_sst.yaml b/bidscoin/heuristics/bidsmap_sst.yaml index 79a2922c..184f7379 100644 --- a/bidscoin/heuristics/bidsmap_sst.yaml +++ b/bidscoin/heuristics/bidsmap_sst.yaml @@ -29,7 +29,6 @@ Options: bidsignore: [extra_data/, sub-*_ct.*] # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv] unknowntypes: [extra_data] # A list of datatypes that are converted to BIDS-like datatype folders ignoretypes: [exclude] # A list of datatypes that are excluded / not converted to BIDS - notderivative: [] # A list of suffixes that should not be considered as a derivative datatype unzip: # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz' plugins: # List of plugins with plugin-specific key-value pairs (that can be used by the plugin) dcm2niix2bids: # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info diff --git a/bidscoin/heuristics/schema.json b/bidscoin/heuristics/schema.json index 3f090135..492a9991 100644 --- a/bidscoin/heuristics/schema.json +++ b/bidscoin/heuristics/schema.json @@ -22,7 +22,6 @@ "sesprefix": { "type": ["string", "null"] }, "unknowntypes": { "type": ["array"] }, "ignoretypes": { "type": ["array"] }, - "notderivative": { "type": ["array"] }, "unzip": { "type": ["string", "null"] } }, "additionalProperties": false diff --git a/bidscoin/plugins/dcm2niix2bids.py b/bidscoin/plugins/dcm2niix2bids.py index a5615540..839f0e1c 100644 --- a/bidscoin/plugins/dcm2niix2bids.py +++ b/bidscoin/plugins/dcm2niix2bids.py @@ -210,7 +210,6 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[N # Get started and see what dataformat we have options = Plugin(bidsmap.plugins['dcm2niix2bids']) - exceptions: list = bidsmap.options.get('notderivative', []) bidsignore: list = bidsmap.options['bidsignore'] fallback = 'fallback' if options.get('fallback','y').lower() in ('y', 'yes', 'true') else '' datasource = bids.get_datasource(session, Plugins({'dcm2niix2bids': options})) @@ -268,11 +267,8 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[N LOGGER.info(f"--> Coining: {run.datasource}") # Create the BIDS session/datatype output folder - suffix = run.datasource.dynamicvalue(run.bids['suffix'], True, True) - if suffix in bids.get_derivatives(run.datatype, exceptions): - outfolder = bidsfolder/'derivatives'/manufacturer.replace(' ','')/subid/sesid/run.datatype - else: - outfolder = bidsses/run.datatype + suffix = run.datasource.dynamicvalue(run.bids['suffix'], True, True) + outfolder = bidsses/run.datatype outfolder.mkdir(parents=True, exist_ok=True) # Compose the BIDS filename using the matched run @@ -455,13 +451,9 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[N # Write out provenance data bids.bidsprov(bidsses, source, run, targets) - # Loop over all non-derivative targets (i.e. the produced output files) and edit the json sidecar data + # Loop over all targets (i.e. the produced output files) and edit the json sidecar data for target in sorted(targets): - # Editing derivative data is out of our scope - if target.relative_to(bidsfolder).parts[0] == 'derivatives': - continue - # Load/copy over the source meta-data jsonfile = target.with_suffix('').with_suffix('.json') if not jsonfile.is_file(): diff --git a/bidscoin/plugins/nibabel2bids.py b/bidscoin/plugins/nibabel2bids.py index 37a76a04..32752832 100644 --- a/bidscoin/plugins/nibabel2bids.py +++ b/bidscoin/plugins/nibabel2bids.py @@ -234,9 +234,8 @@ def bidscoiner_plugin(session: Path, bidsmap: BidsMap, bidsses: Path) -> None: json.dump(metadata, json_fid, indent=4) # Add an entry to the scans_table (we typically don't have useful data to put there) - if 'derivatives' not in bidsses.parts: - acq_time = dateutil.parser.parse(f"1925-01-01T{metadata.get('AcquisitionTime', '')}") - scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time.isoformat() + acq_time = dateutil.parser.parse(f"1925-01-01T{metadata.get('AcquisitionTime', '')}") + scans_table.loc[target.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time.isoformat() if not runid: LOGGER.info(f"--> No {__name__} sourcedata found in: {session}") diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ba492c6a..0b90fcc6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,9 @@ ## [dev] +### Changed +- Drop saving data in the derivatives folder (i.e. this was not required by BIDS after all) + ## [4.4.0] - 2024-10-02 ### Added diff --git a/docs/options.rst b/docs/options.rst index 8f532f37..b39537f6 100644 --- a/docs/options.rst +++ b/docs/options.rst @@ -23,7 +23,6 @@ These setting can be used by all the BIDScoin tools: - ``datatypes``: Datatypes that are converted to BIDS. This can be useful for ignoring/excluding specific data types (without changing their mappings) - ``unknowntypes``: Datatypes that are not part of BIDS but that are converted to a BIDS-like entries in the BIDS folder - ``ignoretypes``: Datatypes that are excluded/not converted""" -- ``notderivative``: A list of suffixes that should not be considered as a derivative data type (i.e. use this to make exceptions for what goes into ``bids/derivatives``) - ``unzip``: Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data. Use for instance '\*.tar.gz' if your source data looks like ``sub-01\01_MPRAGE\dcmfiles.tar.gz``, etc The core working of BIDScoin and its plugins can be tested by clicking the corresponding [Test] button and inspection of the terminal output. diff --git a/tests/test_data/bidsmap.yaml b/tests/test_data/bidsmap.yaml index 4e5cb0f0..75eebe82 100644 --- a/tests/test_data/bidsmap.yaml +++ b/tests/test_data/bidsmap.yaml @@ -28,7 +28,6 @@ Options: sesprefix: ses- # The session prefix of the source data unknowntypes: [mrs, extra_data] # A list of datatypes that are converted to BIDS-like datatype folders ignoretypes: [exclude] # A list of datatypes that are excluded/not converted to BIDS - notderivative: [] # A list of suffixes that should not be considered as a derivative datatype unzip: # Wildcard pattern to select tarballed/zip-files in the sourcefolders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz' plugins: # List of plugins with plugin-specific key-value pairs (that can be used by the plugin) dcm2niix2bids: # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info