MS CAPS MAPS FOR MOOD

.gitignore

@@ -40,3 +40,6 @@ clinicadl/notebooks/
 # Mask and other files
 clinicadl/clinicadl/resources/masks/*.nii
 clinicadl/clinicadl/resources/masks/*.nii.gz
+
+# Jean zay 
+core-rg*

CHANGELOG

@@ -5,6 +5,94 @@ Main changes to this code/ project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## ClinicaDL 1.4.0
+
+### Fixed
+
+* Fix `--diagnoses` and `--merged_tsv` option bug in `clinicadl get-labels`
+* Fix Pathlib bugs 
+* Fix `get_tsv_paths` function bug.
+* Fix a bug for which it was impossible to use predict without specifying the splits and selection metrics.
+
+### Changed
+
+* Changed default batch size to 8 for `clinicadl predict`
+* Changed VAEs main class
+
+### New
+
+* Add `--n_proc` option in `clinicadl generate` pipelines for parallelization.
+* Add `--split` to `clinicadl predict`
+* Add new VAE networks.
+* Add pytorch function to summarize
+* Add `--size_reduction` and `--size_reduction_factor` options to `clinicadl train`, `clinicadl predict` and `clinicadl interpret`.
+* Add SSIM2D, SSIM3D metrics for VAE.
+* Add `--save_latent_space` option to `clinicadl train` and `clinicadl predict`.
+* Add `clinicadl generate trvial_motion`
+* Add Data augmentation with torchio
+
+
+
+## ClinicaDL 1.3.1
+
+### Fixed
+
+* Fix TypeError when running ClinicaDL.
+* Fix `--extract_json` option bug in `clinicadl prepare-data`.
+* Fix `clinicadl tsvtools get-labels` error finding `clinica iotools missing-modalities` output. 
+
+### Changed
+
+* Changed `clinicadl tsvtools get-labels` output directory.
+
+### New 
+
+* Add `--caps_directory` option in `clinicadl tsvtools get-labels`.
+## ClinicaDL 1.3.0
+
+### New 
+
+* Add new command `quality-check pet-linear`.
+* Add new command `generate hypometabolic`.
+* Add new network architecture: `Resnet3D` and `SqueezeExcitationCNN`.
+* Add `flair-linear` modality for `prepare-data` command.
+* Add pytorch profiler.
+* Add `--save_nifti` option for `interpret`command.
+* Add `--output_dir` argument for `tsvtools get-labels` command
+
+### Changed
+
+** Core: **
+
+* Transition from os to pathlib.
+* Update data CI.
+* Improve maps_manager.
+* Change `--acq_label` option for `--tracer`.
+* Update tutorial.
+
+
+## ClinicaDL 1.2.0
+
+### Changed
+
+** Core: **
+
+* Add ClinicaDL installation with pipx.
+* Improve logging.
+* Add method argument to the interpret command to choose between the new Grad-CAM method and the gradient method.
+* Change `extract` command to `prepare-data`.
+* Change output of `get-labels`, `split` and `kfold` commands to one TSV per split instead of one per label.
+* Change `tsvtool` command to `tsvtools`.
+* Change `tsvtools getlabels` command to `tsvtools get-labels` and remove the progression column in the TSV output.
+* Add new commands: `tsvtools get-progression`, `tsvtools get-metadata`, `tsvtools prepare-experiment` (split + kfold), and `tsvtools adapt`.
+* Update data CI.
+* Add a new model for quality check 
+
+### Fixed
+
+- Fix `quality-check t1-linear` 
+
+
 ## ClinicaDL 1.1.1
 
 ### Changed
@@ -18,7 +106,7 @@ greater than 3.8.
 sub-command (before `save_tensor` was also subcommand`).
 - Update some paper references in the documentation.
 - Add a description for the available models.
-- Add the flag `--model_layers` to the `clinicadl train list_models` subcommad
+- Add the flag `--model_layers` to the `clinicadl train list_models` subcommand
 to visualize the model layers.
 
 ### Fixed
@@ -84,7 +172,7 @@ Print help if no argument is given.
 ### Fixed
 
 Fix label code generation
-Remove all occurences of a group in a MAPS when choosing the `overwriting` option.
+Remove all occurrences of a group in a MAPS when choosing the `overwriting` option.
 Fix `resume` when the training.tsv file is empty.
 
 ## ClinicaDL 1.0.3 (release bugfix)
@@ -108,7 +196,7 @@ Fix `resume` when the training.tsv file is empty.
 
 - Use the GPU option on the command line interface.
 - Fix the train task when invoking the CLI.
-- Remplace the `predict` arguments passed through the CLI.
+- Replace the `predict` arguments passed through the CLI.
 - Fix issue when using multiple cohorts.
 
 ## ClinicaDL 1.0.1
@@ -135,12 +223,12 @@ introduction of [Click](https://click.palletsprojects.com/en/8.0.x/) library.
 Major changes in several pipelines interfaces have been implemented. For
 instance the preprocessing pipeline has been split. Train pipeline has also
 been refactored to reduce the number of options. The goal is to make ClinicaDL
-more easy to use, to maintain, and adapt the command line to the MAPS.
+easier to use, to maintain, and adapt the command line to the MAPS.
 
 ClinicaDL aim for more reproducibility: some configuration files are saved in
 the MAPS to reproduce experiments in the same condition (with same
-environnement and same parameters). In addition we added some options to fix
-the random processes seed and use Pytorch latest enhancement for a
+environment and same parameters). In addition, we added some options to fix
+the random processes seed and use Pytorch's latest enhancement for a
 deterministic behavior.
 
 Other improvement and small fix have also been implemented.

clinicadl/abnormality_map/abnormality_map.py

@@ -0,0 +1,163 @@
+import json
+import numpy as np
+import nibabel as nib
+from pathlib import Path
+from copy import deepcopy
+from typing import Tuple, Callable, Union, List
+
+import torch
+from torch.utils.data import DataLoader
+
+from clinicadl.utils.caps_dataset.data import load_data_test, return_dataset, get_transforms
+from clinicadl.utils.maps_manager import MapsManager
+from clinicadl.utils.maps_manager.maps_manager_utils import read_json
+
+
+def residual_fn(
+    x: torch.Tensor,
+    x_hat: torch.Tensor,
+) -> Tuple[torch.Tensor, str]:
+    return x - x_hat, "residual"
+
+# def residual_fn(
+#     x: nib.Nifti1Image,
+#     x_hat: nib.Nifti1Image,
+# ) -> Tuple[nib.Nifti1Image, str]:
+#     affine = x.affine
+#     x_data = x.get_fdata().numpy()
+#     x_hat_data = x_hat.get_fdata().numpy()
+#     return nib.Nifti1Image(x_data - x_hat_data, eye(4)), "residual"
+
+abn_map_dict = {
+    "residual": residual_fn,
+}
+
+def compute_abnormality_map(
+    maps_dir: Path,
+    preprocessing_json: Path,
+    data_group: str,
+    gpu: bool,
+    n_proc: int,
+    batch_size: int,
+    abn_map_fn: Union[str, Callable[[nib.Nifti1Image, nib.Nifti1Image], Tuple[nib.Nifti1Image, str]]] = residual_fn,
+    split_list: List[int] = None,
+    tsv_path: Path = None,
+    selection_metrics: str = None,
+    multi_cohort: bool = False,
+):
+    """
+    Computes the abnormality map for a given split and data group using the model selected with the selection metrics.
+
+    Args:
+        maps_manager (MapsManager): MAPS manager object
+        split_list (list[int]): list of splits to compute the abnormality map on
+        data_group (str): data group to compute the abnormality map on
+        selection_metrics (str): selection metrics to use to select the best model
+        abn_map_fn (Callable[[nib.Nifti1Image, nib.Nifti1Image], Tuple[nib.Nifti1Image, str]], optional): 
+            method to compute the abnormality map. Defaults to residual_fn.
+    """
+
+    maps_manager = MapsManager(maps_dir)
+
+    if not split_list:
+        split_list = maps_manager._find_splits()
+
+    for split in split_list:
+
+        if tsv_path is not None:
+            group_df = load_data_test(
+                tsv_path,
+                "CN",
+                multi_cohort=multi_cohort,
+            )
+
+        group_df, group_parameters = maps_manager.get_group_info(data_group, split)
+
+        if not selection_metrics:
+            split_selection_metrics = maps_manager._find_selection_metrics(split)[0]
+        else:
+            split_selection_metrics = selection_metrics[0]
+
+        preprocessing_dict = read_json(preprocessing_json)
+
+        input_preprocessing_dict = deepcopy(preprocessing_dict)
+        input_pattern = preprocessing_dict["file_type"]["pattern"].replace("_mood.pt", "_input.pt")
+        input_preprocessing_dict["file_type"]["pattern"] = input_pattern
+        input_preprocessing_dict["preprocessing"] = "custom"
+
+        output_preprocessing_dict = deepcopy(preprocessing_dict)
+        output_pattern = preprocessing_dict["file_type"]["pattern"].replace("_mood.pt", "_output.pt")
+        output_preprocessing_dict["file_type"]["pattern"] = output_pattern
+        output_preprocessing_dict["preprocessing"] = "custom"
+
+        # TO DO: code to retrieve transforms
+        # _, all_transforms = get_transforms(
+        #     normalize=maps_manager.normalize,
+        #     data_augmentation=maps_manager.data_augmentation,
+        #     size_reduction=maps_manager.size_reduction,
+        #     size_reduction_factor=maps_manager.size_reduction_factor,
+        # )
+
+        caps_output_dir = Path(maps_dir, f"split-{split}", f"best-{split_selection_metrics}", "CapsOutput")
+
+        input_data = return_dataset(
+            caps_output_dir,
+            group_df,
+            input_preprocessing_dict,
+            all_transformations=None, # TO DO: change this
+            multi_cohort=group_parameters["multi_cohort"],
+        )
+
+        output_data = return_dataset(
+            caps_output_dir,
+            group_df,
+            output_preprocessing_dict,
+            all_transformations=None, # TO DO: change this
+            multi_cohort=group_parameters["multi_cohort"],
+        )
+
+        input_loader = DataLoader(
+            input_data,
+            batch_size=batch_size,
+            shuffle=False,
+            num_workers=n_proc,
+        )
+
+        output_loader = DataLoader(
+            output_data,
+            batch_size=batch_size,
+            shuffle=False,
+            num_workers=n_proc,
+        )
+
+        for i, (input_batch, output_batch) in enumerate(zip(input_loader, output_loader)):
+
+            for idx in range(len(input_batch['participant_id'])):
+
+                assert input_batch['participant_id'][idx] == output_batch['participant_id'][idx]
+
+                participant_id = input_batch['participant_id'][idx]
+                session_id = input_batch['session_id'][idx]
+
+                if isinstance(abn_map_fn, str):
+                    abn_map_fn = abn_map_dict[abn_map_fn]
+
+                abn_map_pt, abn_map_pattern = abn_map_fn(output_batch['data'][idx], input_batch['data'][idx])
+                abn_map_nii = nib.Nifti1Image(abn_map_pt.numpy(), np.eye(4))
+
+                abn_map_pt_path = input_batch['image_path'][idx].replace("_input.pt", f"_{abn_map_pattern}.pt")
+                abn_map_nii_path = Path(caps_output_dir, "subjects", participant_id, session_id, "custom", f"{participant_id}_{session_id}_{abn_map_pattern}.nii.gz")
+
+                torch.save(abn_map_pt, abn_map_pt_path)
+                nib.save(abn_map_nii, abn_map_nii_path)
+
+                print("Saved abnormality map at", abn_map_pt_path)
+                print("Saved abnormality map at", abn_map_nii_path)
+
+        abn_map_preprocessing_dict = deepcopy(preprocessing_dict)
+        abn_map_preprocessing_dict["file_type"]["pattern"] = abn_map_pattern
+
+        abn_map_preprocessing_json = json.dumps(abn_map_preprocessing_dict, skipkeys=True, indent=4)
+        f = open(preprocessing_json.replace(".json", f"_{abn_map_pattern}.json"), "w")
+        f.write(abn_map_preprocessing_json)
+        f.close()

clinicadl/abnormality_map/abnormality_map_cli.py

@@ -0,0 +1,66 @@
+import click
+from pathlib import Path
+
+from clinicadl.utils import cli_param
+
+from clinicadl.abnormality_map.abnormality_map import compute_abnormality_map
+
+@click.command(name="abnormality_map", no_args_is_help=True)
+@cli_param.argument.input_maps
+@cli_param.argument.data_group
+@cli_param.argument.preprocessing_json
+@cli_param.option.participant_list
+@cli_param.option.selection_metrics
+@cli_param.option.split
+@cli_param.option.use_gpu
+@cli_param.option.n_proc
+@cli_param.option.batch_size
+@click.option(
+    "--multi_cohort",
+    type=bool,
+    default=False,
+    is_flag=True,
+    help="""Allow to use multiple CAPS directories.
+            In this case, CAPS_DIRECTORY and PARTICIPANTS_TSV must be paths to TSV files.""",
+)
+@click.option(
+    "--abn_map_type", 
+    type=str,
+    default="residual",
+    help="""Type of abnormality map to compute. Default is residual.""",
+)
+def cli(
+    input_maps_directory,
+    data_group,
+    preprocessing_json,
+    participants_tsv,
+    split,
+    abn_map_type,
+    gpu,
+    n_proc,
+    batch_size,
+    selection_metrics,
+    multi_cohort,
+):
+    """Infer the outputs of a trained model on a test set.
+    INPUT_MAPS_DIRECTORY is the MAPS folder from where the model used for prediction will be loaded.
+    DATA_GROUP is the name of the subjects and sessions list used for the interpretation.
+    """
+    from clinicadl.utils.cmdline_utils import check_gpu
+
+    if gpu:
+        check_gpu()
+
+    compute_abnormality_map(
+        maps_dir=input_maps_directory,
+        abn_map_fn=abn_map_type,
+        data_group=data_group,
+        tsv_path=participants_tsv,
+        gpu=gpu,
+        n_proc=n_proc,
+        batch_size=batch_size,
+        split_list=split,
+        selection_metrics=selection_metrics,
+        multi_cohort=multi_cohort,
+        preprocessing_json=preprocessing_json,
+    )

clinicadl/cmdline.py

@@ -11,6 +11,7 @@
 from clinicadl.train.train_cli import cli as train_cli
 from clinicadl.tsvtools.cli import cli as tsvtools_cli
 from clinicadl.utils.maps_manager.logwriter import setup_logging
+from clinicadl.abnormality_map.abnormality_map_cli import cli as abnormality_map_cli
 
 CONTEXT_SETTINGS = dict(
     # Extend content width to avoid shortening of pipeline help.
@@ -44,6 +45,7 @@ def cli(verbosity):
 cli.add_command(interpret_cli)
 cli.add_command(qc_cli)
 cli.add_command(random_search_cli)
+cli.add_command(abnormality_map_cli)
 
 if __name__ == "__main__":
     cli()