Remove xcp_d_dir and clean datasinks (#1262)

xcp_d/cli/parser.py

@@ -858,9 +858,6 @@ def parse_args(args=None, namespace=None):
     work_dir = config.execution.work_dir
     version = config.environment.version
 
-    if config.execution.xcp_d_dir is None:
-        config.execution.xcp_d_dir = output_dir
-
     # Update the config with an empty dict to trigger initialization of all config
     # sections (we used `init=False` above).
     # This must be done after cleaning the work directory, or we could delete an
@@ -882,7 +879,7 @@ def parse_args(args=None, namespace=None):
         )
 
     # Setup directories
-    config.execution.log_dir = config.execution.xcp_d_dir / "logs"
+    config.execution.log_dir = config.execution.output_dir / "logs"
     # Check and create output and working directories
     config.execution.log_dir.mkdir(exist_ok=True, parents=True)
     output_dir.mkdir(exist_ok=True, parents=True)

xcp_d/cli/run.py

@@ -118,7 +118,7 @@ def main():
             from xcp_d.utils.sentry import process_crashfile
 
             crashfolders = [
-                config.execution.xcp_d_dir / f"sub-{s}" / "log" / config.execution.run_uuid
+                config.execution.output_dir / f"sub-{s}" / "log" / config.execution.run_uuid
                 for s in config.execution.participant_label
             ]
             for crashfolder in crashfolders:
@@ -139,14 +139,14 @@ def main():
             sentry_sdk.capture_message(success_message, level="info")
 
         # Bother users with the boilerplate only iff the workflow went okay.
-        boiler_file = config.execution.xcp_d_dir / "logs" / "CITATION.md"
+        boiler_file = config.execution.output_dir / "logs" / "CITATION.md"
         if boiler_file.exists():
             if config.environment.exec_env in (
                 "apptainer",
                 "docker",
             ):
                 boiler_file = Path("<OUTPUT_PATH>") / boiler_file.relative_to(
-                    config.execution.xcp_d_dir
+                    config.execution.output_dir
                 )
             config.loggers.workflow.log(
                 25,
@@ -160,10 +160,10 @@ def main():
         from xcp_d.reports.core import generate_reports
 
         # Write dataset description before generating reports
-        write_dataset_description(config.execution.fmri_dir, config.execution.xcp_d_dir)
+        write_dataset_description(config.execution.fmri_dir, config.execution.output_dir)
 
         if config.execution.atlases:
-            write_atlas_dataset_description(config.execution.xcp_d_dir / "atlases")
+            write_atlas_dataset_description(config.execution.output_dir / "atlases")
 
         # Generate reports phase
         session_list = (
@@ -173,7 +173,7 @@ def main():
         # Generate reports phase
         failed_reports = generate_reports(
             subject_list=config.execution.participant_label,
-            output_dir=config.execution.xcp_d_dir,
+            output_dir=config.execution.output_dir,
             abcc_qc=config.workflow.abcc_qc,
             run_uuid=config.execution.run_uuid,
             session_list=session_list,

xcp_d/cli/workflow.py

@@ -42,7 +42,7 @@ def build_workflow(config_file, retval):
     msg = check_pipeline_version(
         "XCP-D",
         version,
-        config.execution.xcp_d_dir / "dataset_description.json",
+        config.execution.output_dir / "dataset_description.json",
     )
     if msg is not None:
         build_log.warning(msg)
@@ -72,7 +72,7 @@ def build_workflow(config_file, retval):
 
         failed_reports = generate_reports(
             subject_list=config.execution.participant_label,
-            output_dir=config.execution.xcp_d_dir,
+            output_dir=config.execution.output_dir,
             abcc_qc=config.workflow.abcc_qc,
             run_uuid=config.execution.run_uuid,
             session_list=session_list,
@@ -134,7 +134,7 @@ def build_boilerplate(config_file, workflow):
     from xcp_d import config
 
     config.load(config_file)
-    logs_path = config.execution.xcp_d_dir / "logs"
+    logs_path = config.execution.output_dir / "logs"
     boilerplate = workflow.visit_desc()
     citation_files = {ext: logs_path / f"CITATION.{ext}" for ext in ("bib", "tex", "md", "html")}
 

xcp_d/config.py

@@ -26,7 +26,7 @@
 This module implements the memory structures to keep a consistent, singleton config.
 Settings are passed across processes via filesystem, and a copy of the settings for
 each run and subject is left under
-``<xcp_d_dir>/sub-<participant_id>/log/<run_unique_id>/xcp_d.toml``.
+``<output_dir>/sub-<participant_id>/log/<run_unique_id>/xcp_d.toml``.
 Settings are stored using :abbr:`ToML (Tom's Markup Language)`.
 The module has a :py:func:`~xcp_d.config.to_filename` function to allow writing out
 the settings to hard disk in *ToML* format, which looks like:
@@ -385,8 +385,6 @@ class execution(_Config):
     """Only generate a boilerplate."""
     debug = []
     """Debug mode(s)."""
-    xcp_d_dir = None
-    """Root of XCP-D BIDS Derivatives dataset."""
     fs_license_file = _fs_license
     """An existing file containing a FreeSurfer license."""
     layout = None
@@ -427,7 +425,6 @@ class execution(_Config):
     _paths = (
         "fmri_dir",
         "bids_database_dir",
-        "xcp_d_dir",
         "fs_license_file",
         "layout",
         "log_dir",

xcp_d/data/tests/config.toml

@@ -14,7 +14,6 @@ bids_description_hash = "123456"
 bids_filters = {}
 boilerplate_only = false
 debug = []
-xcp_d_dir = "ds000005/derivatives/xcp_d"
 fs_license_file = "/opt/freesurfer/license.txt"
 log_dir = "/opt/xcp_d"
 log_level = 40

xcp_d/tests/test_cli.py

@@ -475,9 +475,9 @@ def _run_and_generate(test_name, parameters, input_type, test_main=False):
         retval = build_workflow(config_file, retval={})
         xcpd_wf = retval["workflow"]
         xcpd_wf.run(**config.nipype.get_plugin())
-        write_dataset_description(config.execution.fmri_dir, config.execution.xcp_d_dir)
+        write_dataset_description(config.execution.fmri_dir, config.execution.output_dir)
         if config.execution.atlases:
-            write_atlas_dataset_description(config.execution.xcp_d_dir / "atlases")
+            write_atlas_dataset_description(config.execution.output_dir / "atlases")
 
         build_boilerplate(str(config_file), xcpd_wf)
         session_list = (
@@ -487,12 +487,12 @@ def _run_and_generate(test_name, parameters, input_type, test_main=False):
         )
         generate_reports(
             subject_list=config.execution.participant_label,
-            output_dir=config.execution.xcp_d_dir,
+            output_dir=config.execution.output_dir,
             abcc_qc=config.workflow.abcc_qc,
             run_uuid=config.execution.run_uuid,
             session_list=session_list,
         )
 
     output_list_file = os.path.join(get_test_data_path(), f"{test_name}_outputs.txt")
-    check_generated_files(config.execution.xcp_d_dir, output_list_file)
-    check_affines(config.execution.fmri_dir, config.execution.xcp_d_dir, input_type=input_type)
+    check_generated_files(config.execution.output_dir, output_list_file)
+    check_affines(config.execution.fmri_dir, config.execution.output_dir, input_type=input_type)

xcp_d/tests/test_workflows_anatomical.py

@@ -9,6 +9,7 @@
 from xcp_d.tests.tests import mock_config
 from xcp_d.tests.utils import get_nodes
 from xcp_d.workflows import anatomical
+from xcp_d.workflows.base import clean_datasinks
 
 
 @pytest.fixture
@@ -68,9 +69,9 @@ def test_warp_surfaces_to_template_wf(
 
     with mock_config():
         config.nipype.omp_nthreads = 1
+        config.execution.output_dir = tmpdir
 
         wf = anatomical.surface.init_warp_surfaces_to_template_wf(
-            output_dir=tmpdir,
             software="FreeSurfer",
             omp_nthreads=1,
         )
@@ -86,15 +87,16 @@ def test_warp_surfaces_to_template_wf(
         wf.inputs.inputnode.template_to_anat_xfm = pnc_data["template_to_anat_xfm"]
 
         wf.base_dir = tmpdir
+        wf = clean_datasinks(wf)
         wf.run()
 
-    # All of the possible fsLR surfaces should be available.
-    out_anat_dir = os.path.join(tmpdir, "sub-1648798153", "ses-PNC1", "anat")
-    for key, filename in surface_files.items():
-        if "fsLR" in key:
-            out_fname = os.path.basename(filename)
-            out_file = os.path.join(out_anat_dir, out_fname)
-            assert os.path.isfile(out_file), "\n".join(sorted(os.listdir(out_anat_dir)))
+        # All of the possible fsLR surfaces should be available.
+        out_anat_dir = os.path.join(tmpdir, "sub-1648798153", "ses-PNC1", "anat")
+        for key, filename in surface_files.items():
+            if "fsLR" in key:
+                out_fname = os.path.basename(filename)
+                out_file = os.path.join(out_anat_dir, out_fname)
+                assert os.path.isfile(out_file), "\n".join(sorted(os.listdir(tmpdir)))
 
 
 def test_postprocess_anat_wf(ds001419_data, tmp_path_factory):
@@ -107,7 +109,7 @@ def test_postprocess_anat_wf(ds001419_data, tmp_path_factory):
     shutil.copyfile(t1w, t2w)
 
     with mock_config():
-        config.execution.xcp_d_dir = tmpdir
+        config.execution.output_dir = tmpdir
         config.workflow.input_type = "fmriprep"
         config.nipype.omp_nthreads = 1
         config.nipype.mem_gb = 0.1
@@ -123,6 +125,7 @@ def test_postprocess_anat_wf(ds001419_data, tmp_path_factory):
         wf.inputs.inputnode.t1w = t1w
         wf.inputs.inputnode.t2w = t2w
         wf.base_dir = tmpdir
+        wf = clean_datasinks(wf)
         wf_res = wf.run()
 
         wf_nodes = get_nodes(wf_res)

xcp_d/tests/test_workflows_connectivity.py

@@ -17,6 +17,7 @@
 from xcp_d.utils.bids import _get_tr
 from xcp_d.utils.utils import _create_mem_gb, get_std2bold_xfms
 from xcp_d.utils.write_save import read_ndata, write_ndata
+from xcp_d.workflows.base import clean_datasinks
 from xcp_d.workflows.bold.connectivity import (
     init_functional_connectivity_cifti_wf,
     init_functional_connectivity_nifti_wf,
@@ -33,7 +34,7 @@ def test_init_load_atlases_wf_nifti(ds001419_data, tmp_path_factory):
     bold_file = ds001419_data["nifti_file"]
 
     with mock_config():
-        config.execution.xcp_d_dir = tmpdir
+        config.execution.output_dir = tmpdir
         config.workflow.file_format = "nifti"
         config.execution.atlases = ["4S156Parcels", "Glasser"]
         config.nipype.omp_nthreads = 1
@@ -58,7 +59,7 @@ def test_init_load_atlases_wf_cifti(ds001419_data, tmp_path_factory):
     bold_file = ds001419_data["cifti_file"]
 
     with mock_config():
-        config.execution.xcp_d_dir = tmpdir
+        config.execution.output_dir = tmpdir
         config.workflow.file_format = "cifti"
         config.execution.atlases = ["4S156Parcels", "Glasser"]
         config.nipype.omp_nthreads = 1
@@ -70,7 +71,7 @@ def test_init_load_atlases_wf_cifti(ds001419_data, tmp_path_factory):
         load_atlases_wf_res = load_atlases_wf.run()
 
         nodes = get_nodes(load_atlases_wf_res)
-        atlas_names = nodes["load_atlases_wf.ds_atlas"].get_output("out_file")
+        atlas_names = nodes["load_atlases_wf.copy_atlas"].get_output("out_file")
         assert len(atlas_names) == 2
 
 
@@ -137,7 +138,7 @@ def test_init_functional_connectivity_nifti_wf(ds001419_data, tmp_path_factory):
 
     # Let's define the inputs and create the workflow
     with mock_config():
-        config.execution.xcp_d_dir = tmpdir
+        config.execution.output_dir = tmpdir
         config.workflow.bandpass_filter = False
         config.workflow.min_coverage = 0.5
         config.nipype.omp_nthreads = 2
@@ -157,6 +158,7 @@ def test_init_functional_connectivity_nifti_wf(ds001419_data, tmp_path_factory):
         connectivity_wf.inputs.inputnode.atlas_files = warped_atlases
         connectivity_wf.inputs.inputnode.atlas_labels_files = atlas_labels_files
         connectivity_wf.base_dir = tmpdir
+        connectivity_wf = clean_datasinks(connectivity_wf)
         connectivity_wf_res = connectivity_wf.run()
 
         nodes = get_nodes(connectivity_wf_res)
@@ -265,7 +267,7 @@ def test_init_functional_connectivity_cifti_wf(ds001419_data, tmp_path_factory):
 
     # Create the node and a tmpdir to write its results out to
     with mock_config():
-        config.execution.xcp_d_dir = tmpdir
+        config.execution.output_dir = tmpdir
         config.workflow.bandpass_filter = False
         config.workflow.min_coverage = 0.5
         config.nipype.omp_nthreads = 2
@@ -285,6 +287,7 @@ def test_init_functional_connectivity_cifti_wf(ds001419_data, tmp_path_factory):
         connectivity_wf.inputs.inputnode.atlas_files = atlas_files
         connectivity_wf.inputs.inputnode.atlas_labels_files = atlas_labels_files
         connectivity_wf.base_dir = tmpdir
+        connectivity_wf = clean_datasinks(connectivity_wf)
         connectivity_wf_res = connectivity_wf.run()
 
         nodes = get_nodes(connectivity_wf_res)

xcp_d/tests/test_workflows_metrics.py

@@ -12,6 +12,7 @@
 from xcp_d.utils.bids import _get_tr
 from xcp_d.utils.utils import _create_mem_gb
 from xcp_d.utils.write_save import read_ndata, write_ndata
+from xcp_d.workflows.base import clean_datasinks
 from xcp_d.workflows.bold import metrics
 
 
@@ -33,7 +34,7 @@ def test_nifti_alff(ds001419_data, tmp_path_factory):
     mem_gbx = _create_mem_gb(bold_file)
 
     with mock_config():
-        config.execution.xcp_d_dir = tempdir
+        config.execution.output_dir = tempdir
         config.workflow.file_format = "nifti"
         config.workflow.low_pass = 0.08
         config.workflow.high_pass = 0.01
@@ -51,6 +52,7 @@ def test_nifti_alff(ds001419_data, tmp_path_factory):
         alff_wf.base_dir = tempdir
         alff_wf.inputs.inputnode.bold_mask = bold_mask
         alff_wf.inputs.inputnode.denoised_bold = bold_file
+        alff_wf = clean_datasinks(alff_wf)
         compute_alff_res = alff_wf.run()
 
         nodes = get_nodes(compute_alff_res)
@@ -92,6 +94,7 @@ def test_nifti_alff(ds001419_data, tmp_path_factory):
         alff_wf.base_dir = tempdir
         alff_wf.inputs.inputnode.bold_mask = bold_mask
         alff_wf.inputs.inputnode.denoised_bold = filename
+        alff_wf = clean_datasinks(alff_wf)
         compute_alff_res = alff_wf.run()
         nodes = get_nodes(compute_alff_res)
 
@@ -120,7 +123,7 @@ def test_cifti_alff(ds001419_data, tmp_path_factory):
     tempdir = tmp_path_factory.mktemp("test_cifti_alff_01")
 
     with mock_config():
-        config.execution.xcp_d_dir = tempdir
+        config.execution.output_dir = tempdir
         config.workflow.file_format = "cifti"
         config.workflow.low_pass = 0.08
         config.workflow.high_pass = 0.01
@@ -137,6 +140,7 @@ def test_cifti_alff(ds001419_data, tmp_path_factory):
         alff_wf.base_dir = tempdir
         alff_wf.inputs.inputnode.bold_mask = bold_mask
         alff_wf.inputs.inputnode.denoised_bold = bold_file
+        alff_wf = clean_datasinks(alff_wf)
         compute_alff_res = alff_wf.run()
 
         nodes = get_nodes(compute_alff_res)
@@ -213,13 +217,14 @@ def test_nifti_reho(ds001419_data, tmp_path_factory):
 
     # Set up and run the ReHo wf in a tempdir
     with mock_config():
-        config.execution.xcp_d_dir = tempdir
+        config.execution.output_dir = tempdir
         config.nipype.omp_nthreads = 2
 
         reho_wf = metrics.init_reho_nifti_wf(name_source=bold_file, mem_gb=mem_gbx)
         reho_wf.inputs.inputnode.bold_mask = bold_mask
         reho_wf.base_dir = tempdir
         reho_wf.inputs.inputnode.denoised_bold = bold_file
+        reho_wf = clean_datasinks(reho_wf)
         reho_res = reho_wf.run()
 
         nodes = get_nodes(reho_res)
@@ -269,7 +274,7 @@ def test_cifti_reho(ds001419_data, tmp_path_factory):
 
     # Set up and run the ReHo wf in a tempdir
     with mock_config():
-        config.execution.xcp_d_dir = tempdir
+        config.execution.output_dir = tempdir
         config.nipype.omp_nthreads = 2
 
         reho_wf = metrics.init_reho_cifti_wf(
@@ -279,6 +284,7 @@ def test_cifti_reho(ds001419_data, tmp_path_factory):
         )
         reho_wf.base_dir = tempdir
         reho_wf.inputs.inputnode.denoised_bold = orig_bold_file
+        reho_wf = clean_datasinks(reho_wf)
         reho_res = reho_wf.run()
 
         nodes = get_nodes(reho_res)
@@ -303,6 +309,7 @@ def test_cifti_reho(ds001419_data, tmp_path_factory):
         )
         reho_wf.base_dir = tempdir
         reho_wf.inputs.inputnode.denoised_bold = noisy_bold_file
+        reho_wf = clean_datasinks(reho_wf)
         reho_res = reho_wf.run()
 
         nodes = get_nodes(reho_res)