From b26b9a7e13abd18b25f87d8430c9d82a5bc75276 Mon Sep 17 00:00:00 2001 From: camillebrianceau Date: Thu, 30 May 2024 15:39:19 +0200 Subject: [PATCH] reorganize architecture --- clinicadl/{utils => }/callbacks/callbacks.py | 4 +- .../callbacks.py => callbacks/config.py} | 2 +- .../{utils => callbacks}/tracking_exp.py | 0 clinicadl/cmdline.py | 16 +-- .../autoencoder => commandline}/__init__.py | 0 .../{config => commandline}/arguments.py | 0 .../modules_options}/__init__.py | 0 .../modules_options}/callbacks.py | 6 +- .../modules_options}/caps_dataset.py | 0 .../modules_options}/computational.py | 4 +- .../modules_options}/cross_validation.py | 4 +- .../modules_options}/data.py | 4 +- .../modules_options}/dataloader.py | 4 +- .../modules_options}/early_stopping.py | 4 +- .../modules_options}/lr_scheduler.py | 0 .../modules_options}/maps_manager.py | 2 +- .../modules_options}/modality.py | 4 +- .../commandline/modules_options/network.py | 20 +++ .../modules_options}/optimization.py | 6 +- .../modules_options}/optimizer.py | 6 +- .../modules_options}/preprocessing.py | 6 +- .../modules_options}/reproducibility.py | 4 +- .../modules_options}/ssda.py | 4 +- .../modules_options}/transforms.py | 6 +- .../modules_options}/validation.py | 4 +- .../cnn => commandline/pipelines}/__init__.py | 0 .../pipelines/generate/__init__.py | 7 ++ .../pipelines/generate/artifacts}/__init__.py | 0 .../pipelines/generate/artifacts/cli.py} | 2 +- .../pipelines/generate/artifacts/options.py} | 4 +- .../pipelines/generate/cli.py} | 20 ++- .../generate/hypometabolic}/__init__.py | 0 .../pipelines/generate/hypometabolic/cli.py} | 4 +- .../generate/hypometabolic/options.py} | 4 +- .../pipelines/generate/random/__init__.py | 0 .../pipelines/generate/random/cli.py} | 9 +- .../pipelines/generate/random/options.py} | 4 +- .../pipelines/generate/shepplogan/__init__.py | 0 .../pipelines/generate/shepplogan/cli.py} | 4 +- .../pipelines/generate/shepplogan/options.py} | 4 +- .../pipelines/generate/trivial/__init__.py | 0 .../pipelines/generate/trivial/cli.py} | 15 ++- .../pipelines/generate/trivial/options.py} | 4 +- .../pipelines/interpret/__init__.py | 0 .../commandline/pipelines/interpret/cli.py | 50 ++++++++ .../pipelines}/interpret/interpret_cli.py | 16 +-- .../pipelines/interpret/options.py} | 4 +- .../commandline/pipelines/predict/__init__.py | 0 .../pipelines/predict/cli.py} | 14 +-- .../pipelines/predict/options.py} | 4 +- .../pipelines/prepare_data/__init__.py | 0 .../prepare_data/prepare_data_cli.py | 3 +- .../prepare_data_from_bids_cli.py | 3 +- .../commandline/pipelines/train/__init__.py | 3 + .../train/classification/__init__.py | 0 .../pipelines/train/classification/cli.py} | 32 ++--- .../train/classification/options.py} | 18 +-- .../pipelines/train/cli.py} | 12 +- .../pipelines/train/from_json/__init__.py | 1 + .../pipelines/train/from_json/cli.py} | 6 +- .../pipelines/train/list_models/__init__.py | 1 + .../pipelines/train/list_models/cli.py} | 0 .../train/reconstruction/__init__.py | 0 .../pipelines/train/reconstruction/cli.py} | 28 +++-- .../train/reconstruction/options.py} | 14 +-- .../pipelines/train/regression/__init__.py | 0 .../pipelines/train/regression/cli.py | 114 ++++++++++++++++++ .../pipelines/train/regression/options.py} | 14 +-- .../train/regression}/regression_cli.py | 4 +- .../pipelines/train/resume/__init__.py | 0 .../pipelines/train/resume/cli.py} | 6 +- .../pipelines/transfer_learning/__init__.py | 0 .../pipelines/transfer_learning/options.py} | 4 +- clinicadl/config/__init__.py | 18 --- clinicadl/config/config/__init__.py | 14 +-- clinicadl/config/config/data.py | 2 +- .../config/pipelines/task/classification.py | 6 +- .../config/pipelines/task/reconstruction.py | 6 +- .../config/pipelines/task/regression.py | 6 +- clinicadl/config/config/pipelines/train.py | 10 +- clinicadl/config/config/ssda.py | 2 +- clinicadl/{utils => config}/config_utils.py | 0 clinicadl/config/options/generate/__init__.py | 7 -- clinicadl/config/options/model.py | 20 --- clinicadl/config/options/task/__init__.py | 1 - clinicadl/generate/generate_config.py | 10 +- .../generate_param/option_hypometabolic.py | 2 +- clinicadl/interpret/gradients.py | 2 +- clinicadl/{utils => }/network/__init__.py | 0 clinicadl/network/autoencoder/__init__.py | 0 .../network/autoencoder/cnn_transformer.py | 2 +- .../{utils => }/network/autoencoder/models.py | 8 +- clinicadl/{utils => }/network/cnn/SECNN.py | 0 clinicadl/network/cnn/__init__.py | 0 clinicadl/{utils => }/network/cnn/models.py | 10 +- clinicadl/{utils => }/network/cnn/random.py | 4 +- clinicadl/{utils => }/network/cnn/resnet.py | 0 clinicadl/{utils => }/network/cnn/resnet3D.py | 0 .../config/model.py => network/config.py} | 2 +- clinicadl/{utils => }/network/network.py | 0 .../{utils => }/network/network_utils.py | 0 clinicadl/{utils => }/network/sub_network.py | 6 +- clinicadl/network/unet/__init__.py | 0 clinicadl/{utils => }/network/unet/unet.py | 2 +- clinicadl/network/vae/__init__.py | 0 .../{utils => }/network/vae/advanced_CVAE.py | 4 +- clinicadl/{utils => }/network/vae/base_vae.py | 2 +- .../network/vae/convolutional_VAE.py | 4 +- .../{utils => }/network/vae/vae_layers.py | 2 +- .../{utils => }/network/vae/vae_utils.py | 0 .../{utils => }/network/vae/vanilla_vae.py | 4 +- .../config => optimizer}/optimization.py | 0 .../{config/config => optimizer}/optimizer.py | 0 clinicadl/predict/predict_manager.py | 4 +- clinicadl/prepare_data/prepare_data.py | 2 +- .../config.py} | 0 .../{utils => preprocessing}/preprocessing.py | 0 clinicadl/quality_check/pet_linear/utils.py | 2 +- clinicadl/quality_check/t1_linear/utils.py | 2 +- .../random_search/random_search_config.py | 6 +- .../random_search/random_search_utils.py | 2 +- clinicadl/train/from_json/__init__.py | 1 - clinicadl/train/list_models/__init__.py | 1 - clinicadl/train/{resume => }/resume.py | 6 +- clinicadl/train/resume/__init__.py | 1 - clinicadl/train/tasks/__init__.py | 1 - clinicadl/train/{tasks => }/tasks_utils.py | 1 + clinicadl/train/utils.py | 5 +- clinicadl/{train => }/trainer/__init__.py | 0 clinicadl/{train => }/trainer/trainer.py | 4 +- .../{train => }/trainer/trainer_utils.py | 0 .../transforms.py => transforms/config.py} | 0 .../{utils => }/transforms/transforms.py | 0 clinicadl/tsvtools/analysis/analysis.py | 4 +- clinicadl/tsvtools/get_labels/get_labels.py | 6 +- .../tsvtools/get_metadata/get_metadata.py | 2 +- .../get_progression/get_progression.py | 2 +- clinicadl/tsvtools/getlabels/getlabels.py | 6 +- clinicadl/tsvtools/kfold/kfold.py | 2 +- clinicadl/tsvtools/split/split.py | 6 +- .../{utils => tsvtools}/tsvtools_utils.py | 0 clinicadl/utils/caps_dataset/data.py | 2 +- clinicadl/utils/cli_param/argument.py | 1 + clinicadl/utils/cli_param/option.py | 2 +- clinicadl/utils/cli_param/option_group.py | 2 + clinicadl/utils/maps_manager/maps_manager.py | 8 +- .../utils/maps_manager/maps_manager_utils.py | 4 +- .../utils/task_manager/reconstruction.py | 6 +- clinicadl/utils/task_manager/task_manager.py | 2 +- 149 files changed, 470 insertions(+), 321 deletions(-) rename clinicadl/{utils => }/callbacks/callbacks.py (97%) rename clinicadl/{config/config/callbacks.py => callbacks/config.py} (90%) rename clinicadl/{utils => callbacks}/tracking_exp.py (100%) rename clinicadl/{utils/network/autoencoder => commandline}/__init__.py (100%) rename clinicadl/{config => commandline}/arguments.py (100%) rename clinicadl/{config/options => commandline/modules_options}/__init__.py (100%) rename clinicadl/{config/options => commandline/modules_options}/callbacks.py (71%) rename clinicadl/{config/options => commandline/modules_options}/caps_dataset.py (100%) rename clinicadl/{config/options => commandline/modules_options}/computational.py (84%) rename clinicadl/{config/options => commandline/modules_options}/cross_validation.py (80%) rename clinicadl/{config/options => commandline/modules_options}/data.py (91%) rename clinicadl/{config/options => commandline/modules_options}/dataloader.py (82%) rename clinicadl/{config/options => commandline/modules_options}/early_stopping.py (77%) rename clinicadl/{config/options => commandline/modules_options}/lr_scheduler.py (100%) rename clinicadl/{config/options => commandline/modules_options}/maps_manager.py (88%) rename clinicadl/{config/options => commandline/modules_options}/modality.py (92%) create mode 100644 clinicadl/commandline/modules_options/network.py rename clinicadl/{config/options => commandline/modules_options}/optimization.py (80%) rename clinicadl/{config/options => commandline/modules_options}/optimizer.py (76%) rename clinicadl/{config/options => commandline/modules_options}/preprocessing.py (94%) rename clinicadl/{config/options => commandline/modules_options}/reproducibility.py (90%) rename clinicadl/{config/options => commandline/modules_options}/ssda.py (88%) rename clinicadl/{config/options => commandline/modules_options}/transforms.py (69%) rename clinicadl/{config/options => commandline/modules_options}/validation.py (88%) rename clinicadl/{utils/network/cnn => commandline/pipelines}/__init__.py (100%) create mode 100644 clinicadl/commandline/pipelines/generate/__init__.py rename clinicadl/{utils/network/unet => commandline/pipelines/generate/artifacts}/__init__.py (100%) rename clinicadl/{generate/generate_artifacts_cli.py => commandline/pipelines/generate/artifacts/cli.py} (99%) rename clinicadl/{config/options/generate/artifacts.py => commandline/pipelines/generate/artifacts/options.py} (91%) rename clinicadl/{generate/generate_cli.py => commandline/pipelines/generate/cli.py} (52%) rename clinicadl/{utils/network/vae => commandline/pipelines/generate/hypometabolic}/__init__.py (100%) rename clinicadl/{generate/generate_hypometabolic_cli.py => commandline/pipelines/generate/hypometabolic/cli.py} (98%) rename clinicadl/{config/options/generate/hypometabolic.py => commandline/pipelines/generate/hypometabolic/options.py} (84%) create mode 100644 clinicadl/commandline/pipelines/generate/random/__init__.py rename clinicadl/{generate/generate_random_cli.py => commandline/pipelines/generate/random/cli.py} (99%) rename clinicadl/{config/options/generate/random.py => commandline/pipelines/generate/random/options.py} (77%) create mode 100644 clinicadl/commandline/pipelines/generate/shepplogan/__init__.py rename clinicadl/{generate/generate_shepplogan_cli.py => commandline/pipelines/generate/shepplogan/cli.py} (97%) rename clinicadl/{config/options/generate/shepplogan.py => commandline/pipelines/generate/shepplogan/options.py} (90%) create mode 100644 clinicadl/commandline/pipelines/generate/trivial/__init__.py rename clinicadl/{generate/generate_trivial_cli.py => commandline/pipelines/generate/trivial/cli.py} (98%) rename clinicadl/{config/options/generate/trivial.py => commandline/pipelines/generate/trivial/options.py} (80%) create mode 100644 clinicadl/commandline/pipelines/interpret/__init__.py create mode 100644 clinicadl/commandline/pipelines/interpret/cli.py rename clinicadl/{ => commandline/pipelines}/interpret/interpret_cli.py (84%) rename clinicadl/{config/options/interpret.py => commandline/pipelines/interpret/options.py} (87%) create mode 100644 clinicadl/commandline/pipelines/predict/__init__.py rename clinicadl/{predict/predict_cli.py => commandline/pipelines/predict/cli.py} (92%) rename clinicadl/{config/options/predict.py => commandline/pipelines/predict/options.py} (78%) create mode 100644 clinicadl/commandline/pipelines/prepare_data/__init__.py rename clinicadl/{ => commandline/pipelines}/prepare_data/prepare_data_cli.py (98%) rename clinicadl/{ => commandline/pipelines}/prepare_data/prepare_data_from_bids_cli.py (99%) create mode 100644 clinicadl/commandline/pipelines/train/__init__.py create mode 100644 clinicadl/commandline/pipelines/train/classification/__init__.py rename clinicadl/{train/tasks/classification_cli.py => commandline/pipelines/train/classification/cli.py} (85%) rename clinicadl/{config/options/task/classification.py => commandline/pipelines/train/classification/options.py} (71%) rename clinicadl/{train/train_cli.py => commandline/pipelines/train/cli.py} (54%) create mode 100644 clinicadl/commandline/pipelines/train/from_json/__init__.py rename clinicadl/{train/from_json/from_json_cli.py => commandline/pipelines/train/from_json/cli.py} (90%) create mode 100644 clinicadl/commandline/pipelines/train/list_models/__init__.py rename clinicadl/{train/list_models/list_models_cli.py => commandline/pipelines/train/list_models/cli.py} (100%) create mode 100644 clinicadl/commandline/pipelines/train/reconstruction/__init__.py rename clinicadl/{train/tasks/reconstruction_cli.py => commandline/pipelines/train/reconstruction/cli.py} (86%) rename clinicadl/{config/options/task/reconstruction.py => commandline/pipelines/train/reconstruction/options.py} (70%) create mode 100644 clinicadl/commandline/pipelines/train/regression/__init__.py create mode 100644 clinicadl/commandline/pipelines/train/regression/cli.py rename clinicadl/{config/options/task/regression.py => commandline/pipelines/train/regression/options.py} (74%) rename clinicadl/{train/tasks => commandline/pipelines/train/regression}/regression_cli.py (96%) create mode 100644 clinicadl/commandline/pipelines/train/resume/__init__.py rename clinicadl/{train/resume/resume_cli.py => commandline/pipelines/train/resume/cli.py} (71%) create mode 100644 clinicadl/commandline/pipelines/transfer_learning/__init__.py rename clinicadl/{config/options/transfer_learning.py => commandline/pipelines/transfer_learning/options.py} (86%) delete mode 100644 clinicadl/config/__init__.py rename clinicadl/{utils => config}/config_utils.py (100%) delete mode 100644 clinicadl/config/options/generate/__init__.py delete mode 100644 clinicadl/config/options/model.py delete mode 100644 clinicadl/config/options/task/__init__.py rename clinicadl/{utils => }/network/__init__.py (100%) create mode 100644 clinicadl/network/autoencoder/__init__.py rename clinicadl/{utils => }/network/autoencoder/cnn_transformer.py (98%) rename clinicadl/{utils => }/network/autoencoder/models.py (90%) rename clinicadl/{utils => }/network/cnn/SECNN.py (100%) create mode 100644 clinicadl/network/cnn/__init__.py rename clinicadl/{utils => }/network/cnn/models.py (97%) rename clinicadl/{utils => }/network/cnn/random.py (98%) rename clinicadl/{utils => }/network/cnn/resnet.py (100%) rename clinicadl/{utils => }/network/cnn/resnet3D.py (100%) rename clinicadl/{config/config/model.py => network/config.py} (92%) rename clinicadl/{utils => }/network/network.py (100%) rename clinicadl/{utils => }/network/network_utils.py (100%) rename clinicadl/{utils => }/network/sub_network.py (98%) create mode 100644 clinicadl/network/unet/__init__.py rename clinicadl/{utils => }/network/unet/unet.py (98%) create mode 100644 clinicadl/network/vae/__init__.py rename clinicadl/{utils => }/network/vae/advanced_CVAE.py (97%) rename clinicadl/{utils => }/network/vae/base_vae.py (97%) rename clinicadl/{utils => }/network/vae/convolutional_VAE.py (99%) rename clinicadl/{utils => }/network/vae/vae_layers.py (99%) rename clinicadl/{utils => }/network/vae/vae_utils.py (100%) rename clinicadl/{utils => }/network/vae/vanilla_vae.py (99%) rename clinicadl/{config/config => optimizer}/optimization.py (100%) rename clinicadl/{config/config => optimizer}/optimizer.py (100%) rename clinicadl/{config/config/preprocessing.py => preprocessing/config.py} (100%) rename clinicadl/{utils => preprocessing}/preprocessing.py (100%) delete mode 100644 clinicadl/train/from_json/__init__.py delete mode 100644 clinicadl/train/list_models/__init__.py rename clinicadl/train/{resume => }/resume.py (94%) delete mode 100644 clinicadl/train/resume/__init__.py delete mode 100644 clinicadl/train/tasks/__init__.py rename clinicadl/train/{tasks => }/tasks_utils.py (95%) rename clinicadl/{train => }/trainer/__init__.py (100%) rename clinicadl/{train => }/trainer/trainer.py (99%) rename clinicadl/{train => }/trainer/trainer_utils.py (100%) rename clinicadl/{config/config/transforms.py => transforms/config.py} (100%) rename clinicadl/{utils => }/transforms/transforms.py (100%) rename clinicadl/{utils => tsvtools}/tsvtools_utils.py (100%) diff --git a/clinicadl/utils/callbacks/callbacks.py b/clinicadl/callbacks/callbacks.py similarity index 97% rename from clinicadl/utils/callbacks/callbacks.py rename to clinicadl/callbacks/callbacks.py index 7415249d4..b59a7ffe0 100644 --- a/clinicadl/utils/callbacks/callbacks.py +++ b/clinicadl/callbacks/callbacks.py @@ -127,14 +127,14 @@ def on_train_end(self, parameters, **kwargs): class Tracker(Callback): def on_train_begin(self, parameters, **kwargs): if parameters["track_exp"] == "wandb": - from clinicadl.utils.tracking_exp import WandB_handler + from clinicadl.callbacks.tracking_exp import WandB_handler self.run = WandB_handler( kwargs["split"], parameters, kwargs["maps_path"].name ) if parameters["track_exp"] == "mlflow": - from clinicadl.utils.tracking_exp import Mlflow_handler + from clinicadl.callbacks.tracking_exp import Mlflow_handler self.run = Mlflow_handler( kwargs["split"], parameters, kwargs["maps_path"].name diff --git a/clinicadl/config/config/callbacks.py b/clinicadl/callbacks/config.py similarity index 90% rename from clinicadl/config/config/callbacks.py rename to clinicadl/callbacks/config.py index 2c0943141..ce6e0ef2e 100644 --- a/clinicadl/config/config/callbacks.py +++ b/clinicadl/callbacks/config.py @@ -7,8 +7,8 @@ from pydantic import BaseModel, ConfigDict, computed_field, field_validator from pydantic.types import NonNegativeFloat, NonNegativeInt, PositiveFloat, PositiveInt +from clinicadl.preprocessing.preprocessing import read_preprocessing from clinicadl.utils.enum import ExperimentTracking -from clinicadl.utils.preprocessing import read_preprocessing logger = getLogger("clinicadl.callbacks_config") diff --git a/clinicadl/utils/tracking_exp.py b/clinicadl/callbacks/tracking_exp.py similarity index 100% rename from clinicadl/utils/tracking_exp.py rename to clinicadl/callbacks/tracking_exp.py diff --git a/clinicadl/cmdline.py b/clinicadl/cmdline.py index 30bae6d78..f38d08ace 100644 --- a/clinicadl/cmdline.py +++ b/clinicadl/cmdline.py @@ -1,17 +1,19 @@ # coding: utf8 import click -from clinicadl.generate.generate_cli import cli as generate_cli -from clinicadl.hugging_face.hugging_face_cli import cli as hf_cli -from clinicadl.interpret.interpret_cli import cli as interpret_cli -from clinicadl.predict.predict_cli import cli as predict_cli -from clinicadl.prepare_data.prepare_data_cli import cli as prepare_data_cli -from clinicadl.prepare_data.prepare_data_from_bids_cli import ( +from clinicadl.commandline.pipelines.generate.cli import cli as generate_cli +from clinicadl.commandline.pipelines.interpret.cli import cli as interpret_cli +from clinicadl.commandline.pipelines.predict.cli import cli as predict_cli +from clinicadl.commandline.pipelines.prepare_data.prepare_data_cli import ( + cli as prepare_data_cli, +) +from clinicadl.commandline.pipelines.prepare_data.prepare_data_from_bids_cli import ( cli as prepare_data_from_bids_cli, ) +from clinicadl.commandline.pipelines.train.cli import cli as train_cli +from clinicadl.hugging_face.hugging_face_cli import cli as hf_cli from clinicadl.quality_check.qc_cli import cli as qc_cli from clinicadl.random_search.random_search_cli import cli as random_search_cli -from clinicadl.train.train_cli import cli as train_cli from clinicadl.tsvtools.cli import cli as tsvtools_cli from clinicadl.utils.logger import setup_logging diff --git a/clinicadl/utils/network/autoencoder/__init__.py b/clinicadl/commandline/__init__.py similarity index 100% rename from clinicadl/utils/network/autoencoder/__init__.py rename to clinicadl/commandline/__init__.py diff --git a/clinicadl/config/arguments.py b/clinicadl/commandline/arguments.py similarity index 100% rename from clinicadl/config/arguments.py rename to clinicadl/commandline/arguments.py diff --git a/clinicadl/config/options/__init__.py b/clinicadl/commandline/modules_options/__init__.py similarity index 100% rename from clinicadl/config/options/__init__.py rename to clinicadl/commandline/modules_options/__init__.py diff --git a/clinicadl/config/options/callbacks.py b/clinicadl/commandline/modules_options/callbacks.py similarity index 71% rename from clinicadl/config/options/callbacks.py rename to clinicadl/commandline/modules_options/callbacks.py index 86f84c6b8..71028ce0d 100644 --- a/clinicadl/config/options/callbacks.py +++ b/clinicadl/commandline/modules_options/callbacks.py @@ -1,8 +1,8 @@ import click -from clinicadl.config.config.callbacks import CallbacksConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.callbacks.config import CallbacksConfig +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type emissions_calculator = click.option( "--calculate_emissions/--dont_calculate_emissions", diff --git a/clinicadl/config/options/caps_dataset.py b/clinicadl/commandline/modules_options/caps_dataset.py similarity index 100% rename from clinicadl/config/options/caps_dataset.py rename to clinicadl/commandline/modules_options/caps_dataset.py diff --git a/clinicadl/config/options/computational.py b/clinicadl/commandline/modules_options/computational.py similarity index 84% rename from clinicadl/config/options/computational.py rename to clinicadl/commandline/modules_options/computational.py index ba8c2adc6..d3c168c47 100644 --- a/clinicadl/config/options/computational.py +++ b/clinicadl/commandline/modules_options/computational.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.computational import ComputationalConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Computational amp = click.option( diff --git a/clinicadl/config/options/cross_validation.py b/clinicadl/commandline/modules_options/cross_validation.py similarity index 80% rename from clinicadl/config/options/cross_validation.py rename to clinicadl/commandline/modules_options/cross_validation.py index ec762a25f..734c86dbb 100644 --- a/clinicadl/config/options/cross_validation.py +++ b/clinicadl/commandline/modules_options/cross_validation.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.cross_validation import CrossValidationConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Cross Validation n_splits = click.option( diff --git a/clinicadl/config/options/data.py b/clinicadl/commandline/modules_options/data.py similarity index 91% rename from clinicadl/config/options/data.py rename to clinicadl/commandline/modules_options/data.py index ae9d95fb9..f242b66c9 100644 --- a/clinicadl/config/options/data.py +++ b/clinicadl/commandline/modules_options/data.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.data import DataConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Data baseline = click.option( diff --git a/clinicadl/config/options/dataloader.py b/clinicadl/commandline/modules_options/dataloader.py similarity index 82% rename from clinicadl/config/options/dataloader.py rename to clinicadl/commandline/modules_options/dataloader.py index 7392f6c0d..5be00322b 100644 --- a/clinicadl/config/options/dataloader.py +++ b/clinicadl/commandline/modules_options/dataloader.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.dataloader import DataLoaderConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # DataLoader batch_size = click.option( diff --git a/clinicadl/config/options/early_stopping.py b/clinicadl/commandline/modules_options/early_stopping.py similarity index 77% rename from clinicadl/config/options/early_stopping.py rename to clinicadl/commandline/modules_options/early_stopping.py index f4e7771e3..6385c3864 100644 --- a/clinicadl/config/options/early_stopping.py +++ b/clinicadl/commandline/modules_options/early_stopping.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.early_stopping import EarlyStoppingConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Early Stopping patience = click.option( diff --git a/clinicadl/config/options/lr_scheduler.py b/clinicadl/commandline/modules_options/lr_scheduler.py similarity index 100% rename from clinicadl/config/options/lr_scheduler.py rename to clinicadl/commandline/modules_options/lr_scheduler.py diff --git a/clinicadl/config/options/maps_manager.py b/clinicadl/commandline/modules_options/maps_manager.py similarity index 88% rename from clinicadl/config/options/maps_manager.py rename to clinicadl/commandline/modules_options/maps_manager.py index 6bc5fb198..a9eeaea89 100644 --- a/clinicadl/config/options/maps_manager.py +++ b/clinicadl/commandline/modules_options/maps_manager.py @@ -1,7 +1,7 @@ import click from clinicadl.config.config.maps_manager import MapsManagerConfig -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_type_from_config_class as get_type maps_dir = click.argument("maps_dir", type=get_type("maps_dir", MapsManagerConfig)) data_group = click.option("data_group", type=get_type("data_group", MapsManagerConfig)) diff --git a/clinicadl/config/options/modality.py b/clinicadl/commandline/modules_options/modality.py similarity index 92% rename from clinicadl/config/options/modality.py rename to clinicadl/commandline/modules_options/modality.py index fc2118db6..9ab94976e 100644 --- a/clinicadl/config/options/modality.py +++ b/clinicadl/commandline/modules_options/modality.py @@ -5,8 +5,8 @@ DTIModalityConfig, PETModalityConfig, ) -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type tracer = click.option( "--tracer", diff --git a/clinicadl/commandline/modules_options/network.py b/clinicadl/commandline/modules_options/network.py new file mode 100644 index 000000000..995ea6ccc --- /dev/null +++ b/clinicadl/commandline/modules_options/network.py @@ -0,0 +1,20 @@ +import click + +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type +from clinicadl.network.config import NetworkConfig + +# Model +multi_network = click.option( + "--multi_network/--single_network", + default=get_default("multi_network", NetworkConfig), + help="If provided uses a multi-network framework.", + show_default=True, +) +dropout = click.option( + "--dropout", + type=get_type("dropout", NetworkConfig), + default=get_default("dropout", NetworkConfig), + help="Rate value applied to dropout layers in a CNN architecture.", + show_default=True, +) diff --git a/clinicadl/config/options/optimization.py b/clinicadl/commandline/modules_options/optimization.py similarity index 80% rename from clinicadl/config/options/optimization.py rename to clinicadl/commandline/modules_options/optimization.py index 82fec05ba..fd88dc06e 100644 --- a/clinicadl/config/options/optimization.py +++ b/clinicadl/commandline/modules_options/optimization.py @@ -1,8 +1,8 @@ import click -from clinicadl.config.config.optimization import OptimizationConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type +from clinicadl.optimizer.optimization import OptimizationConfig # Optimization accumulation_steps = click.option( diff --git a/clinicadl/config/options/optimizer.py b/clinicadl/commandline/modules_options/optimizer.py similarity index 76% rename from clinicadl/config/options/optimizer.py rename to clinicadl/commandline/modules_options/optimizer.py index 6143ac0de..57e3903e3 100644 --- a/clinicadl/config/options/optimizer.py +++ b/clinicadl/commandline/modules_options/optimizer.py @@ -1,8 +1,8 @@ import click -from clinicadl.config.config.optimizer import OptimizerConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type +from clinicadl.optimizer.optimizer import OptimizerConfig # Optimizer learning_rate = click.option( diff --git a/clinicadl/config/options/preprocessing.py b/clinicadl/commandline/modules_options/preprocessing.py similarity index 94% rename from clinicadl/config/options/preprocessing.py rename to clinicadl/commandline/modules_options/preprocessing.py index 470f650f7..62f9bc382 100644 --- a/clinicadl/config/options/preprocessing.py +++ b/clinicadl/commandline/modules_options/preprocessing.py @@ -1,8 +1,8 @@ import click -from clinicadl.config.config.preprocessing import PreprocessingConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type +from clinicadl.preprocessing.config import PreprocessingConfig extract_json = click.option( "-ej", diff --git a/clinicadl/config/options/reproducibility.py b/clinicadl/commandline/modules_options/reproducibility.py similarity index 90% rename from clinicadl/config/options/reproducibility.py rename to clinicadl/commandline/modules_options/reproducibility.py index 2e897607f..b161c0c73 100644 --- a/clinicadl/config/options/reproducibility.py +++ b/clinicadl/commandline/modules_options/reproducibility.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.reproducibility import ReproducibilityConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Reproducibility compensation = click.option( diff --git a/clinicadl/config/options/ssda.py b/clinicadl/commandline/modules_options/ssda.py similarity index 88% rename from clinicadl/config/options/ssda.py rename to clinicadl/commandline/modules_options/ssda.py index 8d5865311..8119726ef 100644 --- a/clinicadl/config/options/ssda.py +++ b/clinicadl/commandline/modules_options/ssda.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.ssda import SSDAConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # SSDA caps_target = click.option( diff --git a/clinicadl/config/options/transforms.py b/clinicadl/commandline/modules_options/transforms.py similarity index 69% rename from clinicadl/config/options/transforms.py rename to clinicadl/commandline/modules_options/transforms.py index 8e886d7e0..7cdc3531d 100644 --- a/clinicadl/config/options/transforms.py +++ b/clinicadl/commandline/modules_options/transforms.py @@ -1,8 +1,8 @@ import click -from clinicadl.config.config.transforms import TransformsConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type +from clinicadl.transforms.config import TransformsConfig # Transform data_augmentation = click.option( diff --git a/clinicadl/config/options/validation.py b/clinicadl/commandline/modules_options/validation.py similarity index 88% rename from clinicadl/config/options/validation.py rename to clinicadl/commandline/modules_options/validation.py index 0fc64abd8..9f26d3311 100644 --- a/clinicadl/config/options/validation.py +++ b/clinicadl/commandline/modules_options/validation.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.validation import ValidationConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Validation valid_longitudinal = click.option( diff --git a/clinicadl/utils/network/cnn/__init__.py b/clinicadl/commandline/pipelines/__init__.py similarity index 100% rename from clinicadl/utils/network/cnn/__init__.py rename to clinicadl/commandline/pipelines/__init__.py diff --git a/clinicadl/commandline/pipelines/generate/__init__.py b/clinicadl/commandline/pipelines/generate/__init__.py new file mode 100644 index 000000000..5bfff41ce --- /dev/null +++ b/clinicadl/commandline/pipelines/generate/__init__.py @@ -0,0 +1,7 @@ +from . import ( + random, + shepplogan, + trivial, +) +from .artifacts import options +from .hypometabolic import options diff --git a/clinicadl/utils/network/unet/__init__.py b/clinicadl/commandline/pipelines/generate/artifacts/__init__.py similarity index 100% rename from clinicadl/utils/network/unet/__init__.py rename to clinicadl/commandline/pipelines/generate/artifacts/__init__.py diff --git a/clinicadl/generate/generate_artifacts_cli.py b/clinicadl/commandline/pipelines/generate/artifacts/cli.py similarity index 99% rename from clinicadl/generate/generate_artifacts_cli.py rename to clinicadl/commandline/pipelines/generate/artifacts/cli.py index 1005fb46c..df3ddd92a 100644 --- a/clinicadl/generate/generate_artifacts_cli.py +++ b/clinicadl/commandline/pipelines/generate/artifacts/cli.py @@ -12,7 +12,7 @@ from clinicadl.utils.clinica_utils import clinicadl_file_reader from clinicadl.utils.maps_manager.iotools import commandline_to_json -from .generate_utils import ( +from .....generate.generate_utils import ( find_file_type, load_and_check_tsv, write_missing_mods, diff --git a/clinicadl/config/options/generate/artifacts.py b/clinicadl/commandline/pipelines/generate/artifacts/options.py similarity index 91% rename from clinicadl/config/options/generate/artifacts.py rename to clinicadl/commandline/pipelines/generate/artifacts/options.py index ff7ff1a44..38fcd7d91 100644 --- a/clinicadl/config/options/generate/artifacts.py +++ b/clinicadl/commandline/pipelines/generate/artifacts/options.py @@ -1,8 +1,8 @@ import click +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type from clinicadl.generate.generate_config import GenerateArtifactsConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type contrast = click.option( "--contrast/--no-contrast", diff --git a/clinicadl/generate/generate_cli.py b/clinicadl/commandline/pipelines/generate/cli.py similarity index 52% rename from clinicadl/generate/generate_cli.py rename to clinicadl/commandline/pipelines/generate/cli.py index 437eed11f..69b2c7bdd 100644 --- a/clinicadl/generate/generate_cli.py +++ b/clinicadl/commandline/pipelines/generate/cli.py @@ -1,10 +1,20 @@ import click -from clinicadl.generate.generate_artifacts_cli import cli as generate_artifacts_cli -from clinicadl.generate.generate_hypometabolic_cli import cli as generate_hypo_cli -from clinicadl.generate.generate_random_cli import cli as generate_random_cli -from clinicadl.generate.generate_shepplogan_cli import cli as generate_shepplogan_cli -from clinicadl.generate.generate_trivial_cli import cli as generate_trivial_cli +from clinicadl.commandline.pipelines.generate.artifacts.cli import ( + cli as generate_artifacts_cli, +) +from clinicadl.commandline.pipelines.generate.hypometabolic.cli import ( + cli as generate_hypo_cli, +) +from clinicadl.commandline.pipelines.generate.random.cli import ( + cli as generate_random_cli, +) +from clinicadl.commandline.pipelines.generate.shepplogan.cli import ( + cli as generate_shepplogan_cli, +) +from clinicadl.commandline.pipelines.generate.trivial.cli import ( + cli as generate_trivial_cli, +) class RegistrationOrderGroup(click.Group): diff --git a/clinicadl/utils/network/vae/__init__.py b/clinicadl/commandline/pipelines/generate/hypometabolic/__init__.py similarity index 100% rename from clinicadl/utils/network/vae/__init__.py rename to clinicadl/commandline/pipelines/generate/hypometabolic/__init__.py diff --git a/clinicadl/generate/generate_hypometabolic_cli.py b/clinicadl/commandline/pipelines/generate/hypometabolic/cli.py similarity index 98% rename from clinicadl/generate/generate_hypometabolic_cli.py rename to clinicadl/commandline/pipelines/generate/hypometabolic/cli.py index 40c060340..c402091ea 100644 --- a/clinicadl/generate/generate_hypometabolic_cli.py +++ b/clinicadl/commandline/pipelines/generate/hypometabolic/cli.py @@ -9,6 +9,7 @@ from clinicadl.generate import generate_param from clinicadl.generate.generate_config import GenerateHypometabolicConfig +from clinicadl.tsvtools.tsvtools_utils import extract_baseline from clinicadl.utils.caps_dataset.data import CapsDataset from clinicadl.utils.clinica_utils import ( RemoteFileStructure, @@ -22,9 +23,8 @@ ) from clinicadl.utils.exceptions import DownloadError from clinicadl.utils.maps_manager.iotools import commandline_to_json -from clinicadl.utils.tsvtools_utils import extract_baseline -from .generate_utils import ( +from .....generate.generate_utils import ( find_file_type, load_and_check_tsv, mask_processing, diff --git a/clinicadl/config/options/generate/hypometabolic.py b/clinicadl/commandline/pipelines/generate/hypometabolic/options.py similarity index 84% rename from clinicadl/config/options/generate/hypometabolic.py rename to clinicadl/commandline/pipelines/generate/hypometabolic/options.py index 6aa6b1f1c..aab2165e7 100644 --- a/clinicadl/config/options/generate/hypometabolic.py +++ b/clinicadl/commandline/pipelines/generate/hypometabolic/options.py @@ -1,8 +1,8 @@ import click +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type from clinicadl.generate.generate_config import GenerateHypometabolicConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type pathology = click.option( "--pathology", diff --git a/clinicadl/commandline/pipelines/generate/random/__init__.py b/clinicadl/commandline/pipelines/generate/random/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/generate/generate_random_cli.py b/clinicadl/commandline/pipelines/generate/random/cli.py similarity index 99% rename from clinicadl/generate/generate_random_cli.py rename to clinicadl/commandline/pipelines/generate/random/cli.py index e084a328c..188cec411 100644 --- a/clinicadl/generate/generate_random_cli.py +++ b/clinicadl/commandline/pipelines/generate/random/cli.py @@ -9,15 +9,14 @@ from clinicadl.generate import generate_param from clinicadl.generate.generate_config import GenerateRandomConfig -from clinicadl.utils.caps_dataset.data import CapsDataset -from clinicadl.utils.clinica_utils import clinicadl_file_reader -from clinicadl.utils.maps_manager.iotools import commandline_to_json - -from .generate_utils import ( +from clinicadl.generate.generate_utils import ( find_file_type, load_and_check_tsv, write_missing_mods, ) +from clinicadl.utils.caps_dataset.data import CapsDataset +from clinicadl.utils.clinica_utils import clinicadl_file_reader +from clinicadl.utils.maps_manager.iotools import commandline_to_json logger = getLogger("clinicadl.generate.random") diff --git a/clinicadl/config/options/generate/random.py b/clinicadl/commandline/pipelines/generate/random/options.py similarity index 77% rename from clinicadl/config/options/generate/random.py rename to clinicadl/commandline/pipelines/generate/random/options.py index 93048ad9d..28a5b24c8 100644 --- a/clinicadl/config/options/generate/random.py +++ b/clinicadl/commandline/pipelines/generate/random/options.py @@ -1,8 +1,8 @@ import click +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type from clinicadl.generate.generate_config import GenerateRandomConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type mean = click.option( "--mean", diff --git a/clinicadl/commandline/pipelines/generate/shepplogan/__init__.py b/clinicadl/commandline/pipelines/generate/shepplogan/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/generate/generate_shepplogan_cli.py b/clinicadl/commandline/pipelines/generate/shepplogan/cli.py similarity index 97% rename from clinicadl/generate/generate_shepplogan_cli.py rename to clinicadl/commandline/pipelines/generate/shepplogan/cli.py index fbb428f46..99cffe574 100644 --- a/clinicadl/generate/generate_shepplogan_cli.py +++ b/clinicadl/commandline/pipelines/generate/shepplogan/cli.py @@ -8,10 +8,10 @@ from clinicadl.generate import generate_param from clinicadl.generate.generate_config import GenerateSheppLoganConfig +from clinicadl.preprocessing.preprocessing import write_preprocessing from clinicadl.utils.maps_manager.iotools import check_and_clean, commandline_to_json -from clinicadl.utils.preprocessing import write_preprocessing -from .generate_utils import ( +from .....generate.generate_utils import ( generate_shepplogan_phantom, write_missing_mods, ) diff --git a/clinicadl/config/options/generate/shepplogan.py b/clinicadl/commandline/pipelines/generate/shepplogan/options.py similarity index 90% rename from clinicadl/config/options/generate/shepplogan.py rename to clinicadl/commandline/pipelines/generate/shepplogan/options.py index 8736d3b30..1fa8ab545 100644 --- a/clinicadl/config/options/generate/shepplogan.py +++ b/clinicadl/commandline/pipelines/generate/shepplogan/options.py @@ -2,9 +2,9 @@ import click +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type from clinicadl.generate.generate_config import GenerateSheppLoganConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type extract_json = click.option( "-ej", diff --git a/clinicadl/commandline/pipelines/generate/trivial/__init__.py b/clinicadl/commandline/pipelines/generate/trivial/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/generate/generate_trivial_cli.py b/clinicadl/commandline/pipelines/generate/trivial/cli.py similarity index 98% rename from clinicadl/generate/generate_trivial_cli.py rename to clinicadl/commandline/pipelines/generate/trivial/cli.py index b27536e5d..0559fa48a 100644 --- a/clinicadl/generate/generate_trivial_cli.py +++ b/clinicadl/commandline/pipelines/generate/trivial/cli.py @@ -9,6 +9,13 @@ from clinicadl.generate import generate_param from clinicadl.generate.generate_config import GenerateTrivialConfig +from clinicadl.generate.generate_utils import ( + find_file_type, + im_loss_roi_gaussian_distribution, + load_and_check_tsv, + write_missing_mods, +) +from clinicadl.tsvtools.tsvtools_utils import extract_baseline from clinicadl.utils.caps_dataset.data import CapsDataset from clinicadl.utils.clinica_utils import ( RemoteFileStructure, @@ -16,14 +23,6 @@ fetch_file, ) from clinicadl.utils.maps_manager.iotools import commandline_to_json -from clinicadl.utils.tsvtools_utils import extract_baseline - -from .generate_utils import ( - find_file_type, - im_loss_roi_gaussian_distribution, - load_and_check_tsv, - write_missing_mods, -) logger = getLogger("clinicadl.generate.trivial") diff --git a/clinicadl/config/options/generate/trivial.py b/clinicadl/commandline/pipelines/generate/trivial/options.py similarity index 80% rename from clinicadl/config/options/generate/trivial.py rename to clinicadl/commandline/pipelines/generate/trivial/options.py index 56133907a..e92332f5b 100644 --- a/clinicadl/config/options/generate/trivial.py +++ b/clinicadl/commandline/pipelines/generate/trivial/options.py @@ -2,9 +2,9 @@ import click +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type from clinicadl.generate.generate_config import GenerateTrivialConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type mask_path = click.option( "--mask_path", diff --git a/clinicadl/commandline/pipelines/interpret/__init__.py b/clinicadl/commandline/pipelines/interpret/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/commandline/pipelines/interpret/cli.py b/clinicadl/commandline/pipelines/interpret/cli.py new file mode 100644 index 000000000..95ab859df --- /dev/null +++ b/clinicadl/commandline/pipelines/interpret/cli.py @@ -0,0 +1,50 @@ +import click + +from clinicadl.commandline import arguments +from clinicadl.commandline.modules_options import ( + computational, + data, + dataloader, + maps_manager, + validation, +) +from clinicadl.commandline.pipelines.interpret import options +from clinicadl.config.config.pipelines.interpret import InterpretConfig +from clinicadl.predict.predict_manager import PredictManager + + +@click.command("interpret", no_args_is_help=True) +@arguments.input_maps +@arguments.data_group +@maps_manager.overwrite +@maps_manager.save_nifti +@options.name +@options.method +@options.level +@options.target_node +@options.save_individual +@options.overwrite_name +@data.participants_tsv +@data.caps_directory +@data.multi_cohort +@data.diagnoses +@dataloader.n_proc +@dataloader.batch_size +@computational.gpu +@computational.amp +@validation.selection_metrics +def cli(**kwargs): + """Interpretation of trained models using saliency map method. + INPUT_MAPS_DIRECTORY is the MAPS folder from where the model to interpret will be loaded. + DATA_GROUP is the name of the subjects and sessions list used for the interpretation. + NAME is the name of the saliency map task. + METHOD is the method used to extract an attribution map. + """ + + interpret_config = InterpretConfig(**kwargs) + predict_manager = PredictManager(interpret_config) + predict_manager.interpret() + + +if __name__ == "__main__": + cli() diff --git a/clinicadl/interpret/interpret_cli.py b/clinicadl/commandline/pipelines/interpret/interpret_cli.py similarity index 84% rename from clinicadl/interpret/interpret_cli.py rename to clinicadl/commandline/pipelines/interpret/interpret_cli.py index fce61533a..a2cc8a929 100644 --- a/clinicadl/interpret/interpret_cli.py +++ b/clinicadl/commandline/pipelines/interpret/interpret_cli.py @@ -1,12 +1,12 @@ import click -from clinicadl.config import arguments +from clinicadl.commandline import arguments +from clinicadl.commandline.pipelines.interpret import options from clinicadl.config.config.pipelines.interpret import InterpretConfig from clinicadl.config.options import ( computational, data, dataloader, - interpret, maps_manager, validation, ) @@ -18,12 +18,12 @@ @arguments.data_group @maps_manager.overwrite @maps_manager.save_nifti -@interpret.name -@interpret.method -@interpret.level -@interpret.target_node -@interpret.save_individual -@interpret.overwrite_name +@options.name +@options.method +@options.level +@options.target_node +@options.save_individual +@options.overwrite_name @data.participants_tsv @data.caps_directory @data.multi_cohort diff --git a/clinicadl/config/options/interpret.py b/clinicadl/commandline/pipelines/interpret/options.py similarity index 87% rename from clinicadl/config/options/interpret.py rename to clinicadl/commandline/pipelines/interpret/options.py index 1c34035ab..8dee6d099 100644 --- a/clinicadl/config/options/interpret.py +++ b/clinicadl/commandline/pipelines/interpret/options.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.pipelines.interpret import InterpretConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # interpret specific name = click.argument( diff --git a/clinicadl/commandline/pipelines/predict/__init__.py b/clinicadl/commandline/pipelines/predict/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/predict/predict_cli.py b/clinicadl/commandline/pipelines/predict/cli.py similarity index 92% rename from clinicadl/predict/predict_cli.py rename to clinicadl/commandline/pipelines/predict/cli.py index 411455803..4d2a6a8a0 100644 --- a/clinicadl/predict/predict_cli.py +++ b/clinicadl/commandline/pipelines/predict/cli.py @@ -1,16 +1,16 @@ import click -from clinicadl.config import arguments -from clinicadl.config.config.pipelines.predict import PredictConfig -from clinicadl.config.options import ( +from clinicadl.commandline import arguments +from clinicadl.commandline.modules_options import ( computational, cross_validation, data, dataloader, maps_manager, - predict, validation, ) +from clinicadl.commandline.pipelines.predict import options +from clinicadl.config.config.pipelines.predict import PredictConfig from clinicadl.predict.predict_manager import PredictManager @@ -19,10 +19,10 @@ @arguments.data_group @maps_manager.save_nifti @maps_manager.overwrite -@predict.use_labels +@options.use_labels @data.label -@predict.save_tensor -@predict.save_latent_tensor +@options.save_tensor +@options.save_latent_tensor @data.caps_directory @data.participants_tsv @data.multi_cohort diff --git a/clinicadl/config/options/predict.py b/clinicadl/commandline/pipelines/predict/options.py similarity index 78% rename from clinicadl/config/options/predict.py rename to clinicadl/commandline/pipelines/predict/options.py index 15ab7a6d8..3e7ae7ccb 100644 --- a/clinicadl/config/options/predict.py +++ b/clinicadl/commandline/pipelines/predict/options.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.pipelines.predict import PredictConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # predict specific use_labels = click.option( diff --git a/clinicadl/commandline/pipelines/prepare_data/__init__.py b/clinicadl/commandline/pipelines/prepare_data/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/prepare_data/prepare_data_cli.py b/clinicadl/commandline/pipelines/prepare_data/prepare_data_cli.py similarity index 98% rename from clinicadl/prepare_data/prepare_data_cli.py rename to clinicadl/commandline/pipelines/prepare_data/prepare_data_cli.py index 74e4d7550..540789d0e 100644 --- a/clinicadl/prepare_data/prepare_data_cli.py +++ b/clinicadl/commandline/pipelines/prepare_data/prepare_data_cli.py @@ -4,6 +4,7 @@ import click from clinicadl.prepare_data import prepare_data_param +from clinicadl.prepare_data.prepare_data import DeepLearningPrepareData from clinicadl.prepare_data.prepare_data_config import ( PrepareDataImageConfig, PrepareDataPatchConfig, @@ -21,8 +22,6 @@ Tracer, ) -from .prepare_data import DeepLearningPrepareData - @click.command(name="image", no_args_is_help=True) @prepare_data_param.argument.caps_directory diff --git a/clinicadl/prepare_data/prepare_data_from_bids_cli.py b/clinicadl/commandline/pipelines/prepare_data/prepare_data_from_bids_cli.py similarity index 99% rename from clinicadl/prepare_data/prepare_data_from_bids_cli.py rename to clinicadl/commandline/pipelines/prepare_data/prepare_data_from_bids_cli.py index 63c71b83a..e06ac7f12 100644 --- a/clinicadl/prepare_data/prepare_data_from_bids_cli.py +++ b/clinicadl/commandline/pipelines/prepare_data/prepare_data_from_bids_cli.py @@ -3,10 +3,9 @@ import click +from clinicadl.prepare_data.prepare_data import DeepLearningPrepareData from clinicadl.utils import cli_param -from .prepare_data import DeepLearningPrepareData - @click.command(name="image", no_args_is_help=True) @cli_param.argument.bids_directory diff --git a/clinicadl/commandline/pipelines/train/__init__.py b/clinicadl/commandline/pipelines/train/__init__.py new file mode 100644 index 000000000..ae6937b4f --- /dev/null +++ b/clinicadl/commandline/pipelines/train/__init__.py @@ -0,0 +1,3 @@ +from .classification import options +from .reconstruction import options +from .regression import options diff --git a/clinicadl/commandline/pipelines/train/classification/__init__.py b/clinicadl/commandline/pipelines/train/classification/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/train/tasks/classification_cli.py b/clinicadl/commandline/pipelines/train/classification/cli.py similarity index 85% rename from clinicadl/train/tasks/classification_cli.py rename to clinicadl/commandline/pipelines/train/classification/cli.py index be21f690f..9f0ece917 100644 --- a/clinicadl/train/tasks/classification_cli.py +++ b/clinicadl/commandline/pipelines/train/classification/cli.py @@ -1,8 +1,7 @@ import click -from clinicadl.config import arguments -from clinicadl.config.config.pipelines.task.classification import ClassificationConfig -from clinicadl.config.options import ( +from clinicadl.commandline import arguments +from clinicadl.commandline.modules_options import ( callbacks, computational, cross_validation, @@ -10,18 +9,23 @@ dataloader, early_stopping, lr_scheduler, - model, + network, optimization, optimizer, reproducibility, ssda, - task, - transfer_learning, transforms, validation, ) -from clinicadl.train.trainer import Trainer +from clinicadl.commandline.pipelines.train.classification import ( + options as classification, +) +from clinicadl.commandline.pipelines.transfer_learning import ( + options as transfer_learning, +) +from clinicadl.config.config.pipelines.task.classification import ClassificationConfig from clinicadl.train.utils import merge_cli_and_config_file_options +from clinicadl.trainer.trainer import Trainer from clinicadl.utils.enum import Task @@ -43,8 +47,8 @@ @reproducibility.save_all_models @reproducibility.config_file # Model -@model.dropout -@model.multi_network +@network.dropout +@network.multi_network # Data @data.multi_cohort @data.diagnoses @@ -89,11 +93,11 @@ @callbacks.emissions_calculator @callbacks.track_exp # Task-related -@task.classification.architecture -@task.classification.label -@task.classification.selection_metrics -@task.classification.threshold -@task.classification.loss +@classification.architecture +@classification.label +@classification.selection_metrics +@classification.threshold +@classification.loss def cli(**kwargs): """ Train a deep learning model to learn a classification task on neuroimaging data. diff --git a/clinicadl/config/options/task/classification.py b/clinicadl/commandline/pipelines/train/classification/options.py similarity index 71% rename from clinicadl/config/options/task/classification.py rename to clinicadl/commandline/pipelines/train/classification/options.py index f46e6d521..31e558ee3 100644 --- a/clinicadl/config/options/task/classification.py +++ b/clinicadl/commandline/pipelines/train/classification/options.py @@ -2,11 +2,11 @@ from clinicadl.config.config.pipelines.task.classification import ( DataConfig, - ModelConfig, + NetworkConfig, ValidationConfig, ) -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Data label = click.option( @@ -20,22 +20,22 @@ architecture = click.option( "-a", "--architecture", - type=get_type("architecture", ModelConfig), - default=get_default("architecture", ModelConfig), + type=get_type("architecture", NetworkConfig), + default=get_default("architecture", NetworkConfig), help="Architecture of the chosen model to train. A set of model is available in ClinicaDL, default architecture depends on the NETWORK_TASK (see the documentation for more information).", ) loss = click.option( "--loss", "-l", - type=get_type("loss", ModelConfig), - default=get_default("loss", ModelConfig), + type=get_type("loss", NetworkConfig), + default=get_default("loss", NetworkConfig), help="Loss used by the network to optimize its training task.", show_default=True, ) threshold = click.option( "--selection_threshold", - type=get_type("selection_threshold", ModelConfig), - default=get_default("selection_threshold", ModelConfig), + type=get_type("selection_threshold", NetworkConfig), + default=get_default("selection_threshold", NetworkConfig), help="""Selection threshold for soft-voting. Will only be used if num_networks > 1.""", show_default=True, ) diff --git a/clinicadl/train/train_cli.py b/clinicadl/commandline/pipelines/train/cli.py similarity index 54% rename from clinicadl/train/train_cli.py rename to clinicadl/commandline/pipelines/train/cli.py index a773ac076..8e536efd2 100644 --- a/clinicadl/train/train_cli.py +++ b/clinicadl/commandline/pipelines/train/cli.py @@ -1,11 +1,11 @@ import click -from .from_json import cli as from_json_cli -from .list_models import cli as list_models_cli -from .resume import cli as resume_cli -from .tasks.classification_cli import cli as classification_cli -from .tasks.reconstruction_cli import cli as reconstruction_cli -from .tasks.regression_cli import cli as regression_cli +from .classification.cli import cli as classification_cli +from .from_json.cli import cli as from_json_cli +from .list_models.cli import cli as list_models_cli +from .reconstruction.cli import cli as reconstruction_cli +from .regression.cli import cli as regression_cli +from .resume.cli import cli as resume_cli @click.group(name="train", no_args_is_help=True) diff --git a/clinicadl/commandline/pipelines/train/from_json/__init__.py b/clinicadl/commandline/pipelines/train/from_json/__init__.py new file mode 100644 index 000000000..4b7029b4e --- /dev/null +++ b/clinicadl/commandline/pipelines/train/from_json/__init__.py @@ -0,0 +1 @@ +from .cli import cli diff --git a/clinicadl/train/from_json/from_json_cli.py b/clinicadl/commandline/pipelines/train/from_json/cli.py similarity index 90% rename from clinicadl/train/from_json/from_json_cli.py rename to clinicadl/commandline/pipelines/train/from_json/cli.py index 404b346cd..4fef6896a 100644 --- a/clinicadl/train/from_json/from_json_cli.py +++ b/clinicadl/commandline/pipelines/train/from_json/cli.py @@ -3,12 +3,12 @@ import click -from clinicadl.config import arguments -from clinicadl.config.options import ( +from clinicadl.commandline import arguments +from clinicadl.commandline.modules_options import ( cross_validation, reproducibility, ) -from clinicadl.train.tasks.tasks_utils import create_training_config +from clinicadl.train.tasks_utils import create_training_config @click.command(name="from_json", no_args_is_help=True) diff --git a/clinicadl/commandline/pipelines/train/list_models/__init__.py b/clinicadl/commandline/pipelines/train/list_models/__init__.py new file mode 100644 index 000000000..4b7029b4e --- /dev/null +++ b/clinicadl/commandline/pipelines/train/list_models/__init__.py @@ -0,0 +1 @@ +from .cli import cli diff --git a/clinicadl/train/list_models/list_models_cli.py b/clinicadl/commandline/pipelines/train/list_models/cli.py similarity index 100% rename from clinicadl/train/list_models/list_models_cli.py rename to clinicadl/commandline/pipelines/train/list_models/cli.py diff --git a/clinicadl/commandline/pipelines/train/reconstruction/__init__.py b/clinicadl/commandline/pipelines/train/reconstruction/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/train/tasks/reconstruction_cli.py b/clinicadl/commandline/pipelines/train/reconstruction/cli.py similarity index 86% rename from clinicadl/train/tasks/reconstruction_cli.py rename to clinicadl/commandline/pipelines/train/reconstruction/cli.py index e2b37b10e..8cdff6a65 100644 --- a/clinicadl/train/tasks/reconstruction_cli.py +++ b/clinicadl/commandline/pipelines/train/reconstruction/cli.py @@ -1,8 +1,7 @@ import click -from clinicadl.config import arguments -from clinicadl.config.config.pipelines.task.reconstruction import ReconstructionConfig -from clinicadl.config.options import ( +from clinicadl.commandline import arguments +from clinicadl.commandline.modules_options import ( callbacks, computational, cross_validation, @@ -10,18 +9,23 @@ dataloader, early_stopping, lr_scheduler, - model, + network, optimization, optimizer, reproducibility, ssda, - task, - transfer_learning, transforms, validation, ) -from clinicadl.train.trainer import Trainer +from clinicadl.commandline.pipelines.train.reconstruction import ( + options as reconstruction, +) +from clinicadl.commandline.pipelines.transfer_learning import ( + options as transfer_learning, +) +from clinicadl.config.config.pipelines.task.reconstruction import ReconstructionConfig from clinicadl.train.utils import merge_cli_and_config_file_options +from clinicadl.trainer.trainer import Trainer from clinicadl.utils.enum import Task @@ -43,8 +47,8 @@ @reproducibility.save_all_models @reproducibility.config_file # Model -@model.dropout -@model.multi_network +@network.dropout +@network.multi_network # Data @data.multi_cohort @data.diagnoses @@ -89,9 +93,9 @@ @callbacks.emissions_calculator @callbacks.track_exp # Task-related -@task.reconstruction.architecture -@task.reconstruction.selection_metrics -@task.reconstruction.loss +@reconstruction.architecture +@reconstruction.selection_metrics +@reconstruction.loss def cli(**kwargs): """ Train a deep learning model to learn a reconstruction task on neuroimaging data. diff --git a/clinicadl/config/options/task/reconstruction.py b/clinicadl/commandline/pipelines/train/reconstruction/options.py similarity index 70% rename from clinicadl/config/options/task/reconstruction.py rename to clinicadl/commandline/pipelines/train/reconstruction/options.py index 7270c4fe9..11ba83599 100644 --- a/clinicadl/config/options/task/reconstruction.py +++ b/clinicadl/commandline/pipelines/train/reconstruction/options.py @@ -1,25 +1,25 @@ import click from clinicadl.config.config.pipelines.task.reconstruction import ( - ModelConfig, + NetworkConfig, ValidationConfig, ) -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Model architecture = click.option( "-a", "--architecture", - type=get_type("architecture", ModelConfig), - default=get_default("architecture", ModelConfig), + type=get_type("architecture", NetworkConfig), + default=get_default("architecture", NetworkConfig), help="Architecture of the chosen model to train. A set of model is available in ClinicaDL, default architecture depends on the NETWORK_TASK (see the documentation for more information).", ) loss = click.option( "--loss", "-l", - type=get_type("loss", ModelConfig), - default=get_default("loss", ModelConfig), + type=get_type("loss", NetworkConfig), + default=get_default("loss", NetworkConfig), help="Loss used by the network to optimize its training task.", show_default=True, ) diff --git a/clinicadl/commandline/pipelines/train/regression/__init__.py b/clinicadl/commandline/pipelines/train/regression/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/commandline/pipelines/train/regression/cli.py b/clinicadl/commandline/pipelines/train/regression/cli.py new file mode 100644 index 000000000..f877e342d --- /dev/null +++ b/clinicadl/commandline/pipelines/train/regression/cli.py @@ -0,0 +1,114 @@ +import click + +from clinicadl.commandline import arguments +from clinicadl.commandline.modules_options import ( + callbacks, + computational, + cross_validation, + data, + dataloader, + early_stopping, + lr_scheduler, + network, + optimization, + optimizer, + reproducibility, + ssda, + transforms, + validation, +) +from clinicadl.commandline.pipelines.train.regression import options as regression +from clinicadl.commandline.pipelines.transfer_learning import ( + options as transfer_learning, +) +from clinicadl.config.config.pipelines.task.regression import RegressionConfig +from clinicadl.train.utils import merge_cli_and_config_file_options +from clinicadl.trainer.trainer import Trainer +from clinicadl.utils.enum import Task + + +@click.command(name="regression", no_args_is_help=True) +# Mandatory arguments +@arguments.caps_directory +@arguments.preprocessing_json +@arguments.tsv_directory +@arguments.output_maps +# Options +# Computational +@computational.gpu +@computational.fully_sharded_data_parallel +@computational.amp +# Reproducibility +@reproducibility.seed +@reproducibility.deterministic +@reproducibility.compensation +@reproducibility.save_all_models +@reproducibility.config_file +# Model +@network.dropout +@network.multi_network +# Data +@data.multi_cohort +@data.diagnoses +@data.baseline +# validation +@validation.valid_longitudinal +@validation.evaluation_steps +# transforms +@transforms.normalize +@transforms.data_augmentation +# dataloader +@dataloader.batch_size +@dataloader.sampler +@dataloader.n_proc +# ssda o +@ssda.ssda_network +@ssda.caps_target +@ssda.tsv_target_lab +@ssda.tsv_target_unlab +@ssda.preprocessing_json_target +# Cross validation +@cross_validation.n_splits +@cross_validation.split +# Optimization +@optimizer.optimizer +@optimizer.weight_decay +@optimizer.learning_rate +# lr scheduler +@lr_scheduler.adaptive_learning_rate +# early stopping +@early_stopping.patience +@early_stopping.tolerance +# optimization +@optimization.accumulation_steps +@optimization.profiler +@optimization.epochs +# transfer learning +@transfer_learning.transfer_path +@transfer_learning.transfer_selection_metric +@transfer_learning.nb_unfrozen_layer +# callbacks +@callbacks.emissions_calculator +@callbacks.track_exp +# Task-related +@regression.architecture +@regression.label +@regression.selection_metrics +@regression.loss +def cli(**kwargs): + """ + Train a deep learning model to learn a regression task on neuroimaging data. + CAPS_DIRECTORY is the CAPS folder from where tensors will be loaded. + PREPROCESSING_JSON is the name of the JSON file in CAPS_DIRECTORY/tensor_extraction folder where + all information about extraction are stored in order to read the wanted tensors. + TSV_DIRECTORY is a folder were TSV files defining train and validation sets are stored. + OUTPUT_MAPS_DIRECTORY is the path to the MAPS folder where outputs and results will be saved. + Options for this command can be input by declaring argument on the command line or by providing a + configuration file in TOML format. For more details, please visit the documentation: + https://clinicadl.readthedocs.io/en/stable/Train/Introduction/#configuration-file + """ + options = merge_cli_and_config_file_options(Task.REGRESSION, **kwargs) + options["maps_dir"] = options["output_maps_directory"] + config = RegressionConfig(**options) + trainer = Trainer(config) + trainer.train(split_list=config.cross_validation.split, overwrite=True) diff --git a/clinicadl/config/options/task/regression.py b/clinicadl/commandline/pipelines/train/regression/options.py similarity index 74% rename from clinicadl/config/options/task/regression.py rename to clinicadl/commandline/pipelines/train/regression/options.py index af8a14c50..fdc9ef455 100644 --- a/clinicadl/config/options/task/regression.py +++ b/clinicadl/commandline/pipelines/train/regression/options.py @@ -2,11 +2,11 @@ from clinicadl.config.config.pipelines.task.regression import ( DataConfig, - ModelConfig, + NetworkConfig, ValidationConfig, ) -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type # Data label = click.option( @@ -20,15 +20,15 @@ architecture = click.option( "-a", "--architecture", - type=get_type("architecture", ModelConfig), - default=get_default("architecture", ModelConfig), + type=get_type("architecture", NetworkConfig), + default=get_default("architecture", NetworkConfig), help="Architecture of the chosen model to train. A set of model is available in ClinicaDL, default architecture depends on the NETWORK_TASK (see the documentation for more information).", ) loss = click.option( "--loss", "-l", - type=get_type("loss", ModelConfig), - default=get_default("loss", ModelConfig), + type=get_type("loss", NetworkConfig), + default=get_default("loss", NetworkConfig), help="Loss used by the network to optimize its training task.", show_default=True, ) diff --git a/clinicadl/train/tasks/regression_cli.py b/clinicadl/commandline/pipelines/train/regression/regression_cli.py similarity index 96% rename from clinicadl/train/tasks/regression_cli.py rename to clinicadl/commandline/pipelines/train/regression/regression_cli.py index a3252424b..12ca7b522 100644 --- a/clinicadl/train/tasks/regression_cli.py +++ b/clinicadl/commandline/pipelines/train/regression/regression_cli.py @@ -1,6 +1,7 @@ import click -from clinicadl.config import arguments +from clinicadl.commandline import arguments +from clinicadl.commandline.pipelines import transfer_learning from clinicadl.config.config.pipelines.task.regression import RegressionConfig from clinicadl.config.options import ( callbacks, @@ -16,7 +17,6 @@ reproducibility, ssda, task, - transfer_learning, transforms, validation, ) diff --git a/clinicadl/commandline/pipelines/train/resume/__init__.py b/clinicadl/commandline/pipelines/train/resume/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/train/resume/resume_cli.py b/clinicadl/commandline/pipelines/train/resume/cli.py similarity index 71% rename from clinicadl/train/resume/resume_cli.py rename to clinicadl/commandline/pipelines/train/resume/cli.py index 9186c44e9..5b0efaa38 100644 --- a/clinicadl/train/resume/resume_cli.py +++ b/clinicadl/commandline/pipelines/train/resume/cli.py @@ -1,7 +1,7 @@ import click -from clinicadl.config import arguments -from clinicadl.config.options import ( +from clinicadl.commandline import arguments +from clinicadl.commandline.modules_options import ( cross_validation, ) @@ -14,6 +14,6 @@ def cli(input_maps_directory, split): INPUT_MAPS_DIRECTORY is the path to the MAPS folder where training job has started. """ - from .resume import automatic_resume + from ....train.resume.resume import automatic_resume automatic_resume(input_maps_directory, user_split_list=split) diff --git a/clinicadl/commandline/pipelines/transfer_learning/__init__.py b/clinicadl/commandline/pipelines/transfer_learning/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/config/options/transfer_learning.py b/clinicadl/commandline/pipelines/transfer_learning/options.py similarity index 86% rename from clinicadl/config/options/transfer_learning.py rename to clinicadl/commandline/pipelines/transfer_learning/options.py index 89817838e..643b87caa 100644 --- a/clinicadl/config/options/transfer_learning.py +++ b/clinicadl/commandline/pipelines/transfer_learning/options.py @@ -1,8 +1,8 @@ import click from clinicadl.config.config.transfer_learning import TransferLearningConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_default_from_config_class as get_default +from clinicadl.config.config_utils import get_type_from_config_class as get_type nb_unfrozen_layer = click.option( "-nul", diff --git a/clinicadl/config/__init__.py b/clinicadl/config/__init__.py deleted file mode 100644 index b803220be..000000000 --- a/clinicadl/config/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# from .config import ( -# CallbacksConfig, -# ComputationalConfig, -# CrossValidationConfig, -# DataConfig, -# DataLoaderConfig, -# EarlyStoppingConfig, -# LRschedulerConfig, -# MAPSManagerConfig, -# ModelConfig, -# OptimizationConfig, -# OptimizerConfig, -# ReproducibilityConfig, -# SSDAConfig, -# TransferLearningConfig, -# TransformsConfig, -# ValidationConfig, -# ) diff --git a/clinicadl/config/config/__init__.py b/clinicadl/config/config/__init__.py index 4f59e5e04..010b78390 100644 --- a/clinicadl/config/config/__init__.py +++ b/clinicadl/config/config/__init__.py @@ -1,4 +1,5 @@ -from .callbacks import CallbacksConfig +from ...network.config import NetworkConfig +from ...transforms.config import TransformsConfig from .caps_dataset import CapsDatasetConfig from .computational import ComputationalConfig from .cross_validation import CrossValidationConfig @@ -13,18 +14,7 @@ ModalityConfig, PETModalityConfig, ) -from .model import ModelConfig -from .optimization import OptimizationConfig -from .optimizer import OptimizerConfig -from .preprocessing import ( - PreprocessingConfig, - PreprocessingImageConfig, - PreprocessingPatchConfig, - PreprocessingROIConfig, - PreprocessingSliceConfig, -) from .reproducibility import ReproducibilityConfig from .ssda import SSDAConfig from .transfer_learning import TransferLearningConfig -from .transforms import TransformsConfig from .validation import ValidationConfig diff --git a/clinicadl/config/config/data.py b/clinicadl/config/config/data.py index c6004a7e5..a1c24e23b 100644 --- a/clinicadl/config/config/data.py +++ b/clinicadl/config/config/data.py @@ -4,10 +4,10 @@ from pydantic import BaseModel, ConfigDict, computed_field, field_validator +from clinicadl.preprocessing.preprocessing import read_preprocessing from clinicadl.utils.caps_dataset.data import load_data_test from clinicadl.utils.enum import Mode from clinicadl.utils.maps_manager.maps_manager import MapsManager -from clinicadl.utils.preprocessing import read_preprocessing logger = getLogger("clinicadl.data_config") diff --git a/clinicadl/config/config/pipelines/task/classification.py b/clinicadl/config/config/pipelines/task/classification.py index 9b89cf40c..0bd34ad50 100644 --- a/clinicadl/config/config/pipelines/task/classification.py +++ b/clinicadl/config/config/pipelines/task/classification.py @@ -4,7 +4,7 @@ from pydantic import computed_field, field_validator from clinicadl.config.config import DataConfig as BaseDataConfig -from clinicadl.config.config import ModelConfig as BaseModelConfig +from clinicadl.config.config import NetworkConfig as BaseNetworkConfig from clinicadl.config.config import ValidationConfig as BaseValidationConfig from clinicadl.config.config.pipelines.train import TrainConfig from clinicadl.utils.enum import ClassificationLoss, ClassificationMetric, Task @@ -26,7 +26,7 @@ def validator_label_code(cls, v): return v # TODO : check label_code -class ModelConfig(BaseModelConfig): # TODO : put in model module +class NetworkConfig(BaseNetworkConfig): # TODO : put in model module """Config class for classification models.""" architecture: str = "Conv5_FC3" @@ -69,7 +69,7 @@ class ClassificationConfig(TrainConfig): """ data: DataConfig - model: ModelConfig + model: NetworkConfig validation: ValidationConfig @computed_field diff --git a/clinicadl/config/config/pipelines/task/reconstruction.py b/clinicadl/config/config/pipelines/task/reconstruction.py index b1f63b030..a199bffbc 100644 --- a/clinicadl/config/config/pipelines/task/reconstruction.py +++ b/clinicadl/config/config/pipelines/task/reconstruction.py @@ -4,7 +4,7 @@ from pydantic import PositiveFloat, PositiveInt, computed_field, field_validator -from clinicadl.config.config import ModelConfig as BaseModelConfig +from clinicadl.config.config import NetworkConfig as BaseNetworkConfig from clinicadl.config.config import ValidationConfig as BaseValidationConfig from clinicadl.config.config.pipelines.train import TrainConfig from clinicadl.utils.enum import ( @@ -17,7 +17,7 @@ logger = getLogger("clinicadl.reconstruction_config") -class ModelConfig(BaseModelConfig): # TODO : put in model module +class NetworkConfig(BaseNetworkConfig): # TODO : put in model module """Config class for reconstruction models.""" architecture: str = "AE_Conv5_FC3" @@ -58,7 +58,7 @@ class ReconstructionConfig(TrainConfig): - output_maps_directory """ - model: ModelConfig + model: NetworkConfig validation: ValidationConfig @computed_field diff --git a/clinicadl/config/config/pipelines/task/regression.py b/clinicadl/config/config/pipelines/task/regression.py index 1b6b49018..61a659fab 100644 --- a/clinicadl/config/config/pipelines/task/regression.py +++ b/clinicadl/config/config/pipelines/task/regression.py @@ -5,7 +5,7 @@ from pydantic import PositiveFloat, PositiveInt, computed_field, field_validator from clinicadl.config.config import DataConfig as BaseDataConfig -from clinicadl.config.config import ModelConfig as BaseModelConfig +from clinicadl.config.config import NetworkConfig as BaseNetworkConfig from clinicadl.config.config import ValidationConfig as BaseValidationConfig from clinicadl.config.config.pipelines.train import TrainConfig from clinicadl.utils.enum import RegressionLoss, RegressionMetric, Task @@ -24,7 +24,7 @@ def validator_label(cls, v): return v # TODO : check if label in columns -class ModelConfig(BaseModelConfig): # TODO : put in model module +class NetworkConfig(BaseNetworkConfig): # TODO : put in model module """Config class for regression models.""" architecture: str = "Conv5_FC3" @@ -59,7 +59,7 @@ class RegressionConfig(TrainConfig): """ data: DataConfig - model: ModelConfig + model: NetworkConfig validation: ValidationConfig @computed_field diff --git a/clinicadl/config/config/pipelines/train.py b/clinicadl/config/config/pipelines/train.py index 0cd23773b..2faed9dc1 100644 --- a/clinicadl/config/config/pipelines/train.py +++ b/clinicadl/config/config/pipelines/train.py @@ -7,8 +7,8 @@ computed_field, ) +from clinicadl.callbacks.config import CallbacksConfig from clinicadl.config.config import ( - CallbacksConfig, ComputationalConfig, CrossValidationConfig, DataConfig, @@ -16,15 +16,15 @@ EarlyStoppingConfig, LRschedulerConfig, MapsManagerConfig, - ModelConfig, - OptimizationConfig, - OptimizerConfig, ReproducibilityConfig, SSDAConfig, TransferLearningConfig, TransformsConfig, ValidationConfig, ) +from clinicadl.network.config import NetworkConfig +from clinicadl.optimizer.optimization import OptimizationConfig +from clinicadl.optimizer.optimizer import OptimizerConfig from clinicadl.utils.enum import Task logger = getLogger("clinicadl.training_config") @@ -46,7 +46,7 @@ class TrainConfig(BaseModel, ABC): early_stopping: EarlyStoppingConfig lr_scheduler: LRschedulerConfig maps_manager: MapsManagerConfig - model: ModelConfig + model: NetworkConfig optimization: OptimizationConfig optimizer: OptimizerConfig reproducibility: ReproducibilityConfig diff --git a/clinicadl/config/config/ssda.py b/clinicadl/config/config/ssda.py index a94fe127c..7a9db5006 100644 --- a/clinicadl/config/config/ssda.py +++ b/clinicadl/config/config/ssda.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ConfigDict, computed_field -from clinicadl.utils.preprocessing import read_preprocessing +from clinicadl.preprocessing.preprocessing import read_preprocessing logger = getLogger("clinicadl.ssda_config") diff --git a/clinicadl/utils/config_utils.py b/clinicadl/config/config_utils.py similarity index 100% rename from clinicadl/utils/config_utils.py rename to clinicadl/config/config_utils.py diff --git a/clinicadl/config/options/generate/__init__.py b/clinicadl/config/options/generate/__init__.py deleted file mode 100644 index d78b8fc61..000000000 --- a/clinicadl/config/options/generate/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from . import ( - artifacts, - hypometabolic, - random, - shepplogan, - trivial, -) diff --git a/clinicadl/config/options/model.py b/clinicadl/config/options/model.py deleted file mode 100644 index 0d5ab4083..000000000 --- a/clinicadl/config/options/model.py +++ /dev/null @@ -1,20 +0,0 @@ -import click - -from clinicadl.config.config.model import ModelConfig -from clinicadl.utils.config_utils import get_default_from_config_class as get_default -from clinicadl.utils.config_utils import get_type_from_config_class as get_type - -# Model -multi_network = click.option( - "--multi_network/--single_network", - default=get_default("multi_network", ModelConfig), - help="If provided uses a multi-network framework.", - show_default=True, -) -dropout = click.option( - "--dropout", - type=get_type("dropout", ModelConfig), - default=get_default("dropout", ModelConfig), - help="Rate value applied to dropout layers in a CNN architecture.", - show_default=True, -) diff --git a/clinicadl/config/options/task/__init__.py b/clinicadl/config/options/task/__init__.py deleted file mode 100644 index c6653a77c..000000000 --- a/clinicadl/config/options/task/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import classification, reconstruction, regression diff --git a/clinicadl/generate/generate_config.py b/clinicadl/generate/generate_config.py index eab2da2f0..cf66dfed7 100644 --- a/clinicadl/generate/generate_config.py +++ b/clinicadl/generate/generate_config.py @@ -106,17 +106,9 @@ def gamma_validator(cls, v): class GenerateHypometabolicConfig(SharedGenerateConfigOne): anomaly_degree: float = 30.0 - pathology_cls: Pathology = Pathology.AD + pathology: Pathology = Pathology.AD sigma: int = 5 - @property - def pathology(self) -> Pathology: - return self.pathology_cls - - @pathology.setter - def pathology(self, value: Union[str, Pathology]): - self.pathology_cls = Pathology(value) - class GenerateRandomConfig(SharedGenerateConfigTwo): mean: float = 0.0 diff --git a/clinicadl/generate/generate_param/option_hypometabolic.py b/clinicadl/generate/generate_param/option_hypometabolic.py index ecbe60ca8..ab6640a41 100644 --- a/clinicadl/generate/generate_param/option_hypometabolic.py +++ b/clinicadl/generate/generate_param/option_hypometabolic.py @@ -8,7 +8,7 @@ "--pathology", "-p", type=click.Choice(Pathology), - default=config_hypometabolic["pathology_cls"].default.value, + default=config_hypometabolic["pathology"].default.value, help="Pathology applied. To chose in the following list: [ad, bvftd, lvppa, nfvppa, pca, svppa]", show_default=True, ) diff --git a/clinicadl/interpret/gradients.py b/clinicadl/interpret/gradients.py index a2002af99..393e64488 100644 --- a/clinicadl/interpret/gradients.py +++ b/clinicadl/interpret/gradients.py @@ -50,7 +50,7 @@ class GradCam(Gradients): """ def __init__(self, model): - from clinicadl.utils.network.sub_network import CNN + from clinicadl.network.sub_network import CNN super().__init__(model=model) if not isinstance(model, CNN): diff --git a/clinicadl/utils/network/__init__.py b/clinicadl/network/__init__.py similarity index 100% rename from clinicadl/utils/network/__init__.py rename to clinicadl/network/__init__.py diff --git a/clinicadl/network/autoencoder/__init__.py b/clinicadl/network/autoencoder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/utils/network/autoencoder/cnn_transformer.py b/clinicadl/network/autoencoder/cnn_transformer.py similarity index 98% rename from clinicadl/utils/network/autoencoder/cnn_transformer.py rename to clinicadl/network/autoencoder/cnn_transformer.py index 65c9a7bf3..39270d2a1 100644 --- a/clinicadl/utils/network/autoencoder/cnn_transformer.py +++ b/clinicadl/network/autoencoder/cnn_transformer.py @@ -2,7 +2,7 @@ from torch import nn -from clinicadl.utils.network.network_utils import ( +from clinicadl.network.network_utils import ( CropMaxUnpool2d, CropMaxUnpool3d, PadMaxPool2d, diff --git a/clinicadl/utils/network/autoencoder/models.py b/clinicadl/network/autoencoder/models.py similarity index 90% rename from clinicadl/utils/network/autoencoder/models.py rename to clinicadl/network/autoencoder/models.py index 6eedf4be2..1a9912526 100644 --- a/clinicadl/utils/network/autoencoder/models.py +++ b/clinicadl/network/autoencoder/models.py @@ -1,9 +1,9 @@ from torch import nn -from clinicadl.utils.network.autoencoder.cnn_transformer import CNN_Transformer -from clinicadl.utils.network.cnn.models import Conv4_FC3, Conv5_FC3, resnet18 -from clinicadl.utils.network.sub_network import AutoEncoder -from clinicadl.utils.network.vae.vae_layers import ( +from clinicadl.network.autoencoder.cnn_transformer import CNN_Transformer +from clinicadl.network.cnn.models import Conv4_FC3, Conv5_FC3, resnet18 +from clinicadl.network.sub_network import AutoEncoder +from clinicadl.network.vae.vae_layers import ( DecoderLayer3D, EncoderLayer3D, Flatten, diff --git a/clinicadl/utils/network/cnn/SECNN.py b/clinicadl/network/cnn/SECNN.py similarity index 100% rename from clinicadl/utils/network/cnn/SECNN.py rename to clinicadl/network/cnn/SECNN.py diff --git a/clinicadl/network/cnn/__init__.py b/clinicadl/network/cnn/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/utils/network/cnn/models.py b/clinicadl/network/cnn/models.py similarity index 97% rename from clinicadl/utils/network/cnn/models.py rename to clinicadl/network/cnn/models.py index 3b7aa4e80..87d5e3ce5 100644 --- a/clinicadl/utils/network/cnn/models.py +++ b/clinicadl/network/cnn/models.py @@ -4,11 +4,11 @@ from torch import nn from torchvision.models.resnet import BasicBlock -from clinicadl.utils.network.cnn.resnet import ResNetDesigner, model_urls -from clinicadl.utils.network.cnn.resnet3D import ResNetDesigner3D -from clinicadl.utils.network.cnn.SECNN import SECNNDesigner3D -from clinicadl.utils.network.network_utils import PadMaxPool2d, PadMaxPool3d -from clinicadl.utils.network.sub_network import CNN, CNN_SSDA +from clinicadl.network.cnn.resnet import ResNetDesigner, model_urls +from clinicadl.network.cnn.resnet3D import ResNetDesigner3D +from clinicadl.network.cnn.SECNN import SECNNDesigner3D +from clinicadl.network.network_utils import PadMaxPool2d, PadMaxPool3d +from clinicadl.network.sub_network import CNN, CNN_SSDA def get_layers_fn(input_size): diff --git a/clinicadl/utils/network/cnn/random.py b/clinicadl/network/cnn/random.py similarity index 98% rename from clinicadl/utils/network/cnn/random.py rename to clinicadl/network/cnn/random.py index cee08f48c..d04af8602 100644 --- a/clinicadl/utils/network/cnn/random.py +++ b/clinicadl/network/cnn/random.py @@ -1,9 +1,9 @@ import numpy as np +from clinicadl.network.network_utils import * +from clinicadl.network.sub_network import CNN from clinicadl.utils.enum import Normalization from clinicadl.utils.exceptions import ClinicaDLNetworksError -from clinicadl.utils.network.network_utils import * -from clinicadl.utils.network.sub_network import CNN class RandomArchitecture(CNN): diff --git a/clinicadl/utils/network/cnn/resnet.py b/clinicadl/network/cnn/resnet.py similarity index 100% rename from clinicadl/utils/network/cnn/resnet.py rename to clinicadl/network/cnn/resnet.py diff --git a/clinicadl/utils/network/cnn/resnet3D.py b/clinicadl/network/cnn/resnet3D.py similarity index 100% rename from clinicadl/utils/network/cnn/resnet3D.py rename to clinicadl/network/cnn/resnet3D.py diff --git a/clinicadl/config/config/model.py b/clinicadl/network/config.py similarity index 92% rename from clinicadl/config/config/model.py rename to clinicadl/network/config.py index a5df3ba3d..75fb4ed82 100644 --- a/clinicadl/config/config/model.py +++ b/clinicadl/network/config.py @@ -6,7 +6,7 @@ logger = getLogger("clinicadl.model_config") -class ModelConfig(BaseModel): # TODO : put in model module +class NetworkConfig(BaseModel): # TODO : put in model module """ Abstract config class for the model. diff --git a/clinicadl/utils/network/network.py b/clinicadl/network/network.py similarity index 100% rename from clinicadl/utils/network/network.py rename to clinicadl/network/network.py diff --git a/clinicadl/utils/network/network_utils.py b/clinicadl/network/network_utils.py similarity index 100% rename from clinicadl/utils/network/network_utils.py rename to clinicadl/network/network_utils.py diff --git a/clinicadl/utils/network/sub_network.py b/clinicadl/network/sub_network.py similarity index 98% rename from clinicadl/utils/network/sub_network.py rename to clinicadl/network/sub_network.py index 76a835de2..9d17e8600 100644 --- a/clinicadl/utils/network/sub_network.py +++ b/clinicadl/network/sub_network.py @@ -4,15 +4,15 @@ import torch from torch import nn -from clinicadl.utils.exceptions import ClinicaDLNetworksError -from clinicadl.utils.network.network import Network -from clinicadl.utils.network.network_utils import ( +from clinicadl.network.network import Network +from clinicadl.network.network_utils import ( CropMaxUnpool2d, CropMaxUnpool3d, PadMaxPool2d, PadMaxPool3d, ReverseLayerF, ) +from clinicadl.utils.exceptions import ClinicaDLNetworksError logger = getLogger("clinicadl.networks") diff --git a/clinicadl/network/unet/__init__.py b/clinicadl/network/unet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/utils/network/unet/unet.py b/clinicadl/network/unet/unet.py similarity index 98% rename from clinicadl/utils/network/unet/unet.py rename to clinicadl/network/unet/unet.py index ea89e4bbc..3743f13d8 100644 --- a/clinicadl/utils/network/unet/unet.py +++ b/clinicadl/network/unet/unet.py @@ -1,7 +1,7 @@ import torch from torch import nn -from clinicadl.utils.network.network import Network +from clinicadl.network.network import Network class UNetDown(nn.Module): diff --git a/clinicadl/network/vae/__init__.py b/clinicadl/network/vae/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/clinicadl/utils/network/vae/advanced_CVAE.py b/clinicadl/network/vae/advanced_CVAE.py similarity index 97% rename from clinicadl/utils/network/vae/advanced_CVAE.py rename to clinicadl/network/vae/advanced_CVAE.py index cad7bc422..2da43916e 100644 --- a/clinicadl/utils/network/vae/advanced_CVAE.py +++ b/clinicadl/network/vae/advanced_CVAE.py @@ -2,8 +2,8 @@ import torch.nn as nn import torch.nn.functional as F -from clinicadl.utils.network.network import Network -from clinicadl.utils.network.vae.vae_utils import multiply_list +from clinicadl.network.network import Network +from clinicadl.network.vae.vae_utils import multiply_list class CVAE_3D_final_conv(Network): diff --git a/clinicadl/utils/network/vae/base_vae.py b/clinicadl/network/vae/base_vae.py similarity index 97% rename from clinicadl/utils/network/vae/base_vae.py rename to clinicadl/network/vae/base_vae.py index 19f7cf30e..b9ccb4808 100644 --- a/clinicadl/utils/network/vae/base_vae.py +++ b/clinicadl/network/vae/base_vae.py @@ -1,6 +1,6 @@ import torch -from clinicadl.utils.network.network import Network +from clinicadl.network.network import Network class BaseVAE(Network): diff --git a/clinicadl/utils/network/vae/convolutional_VAE.py b/clinicadl/network/vae/convolutional_VAE.py similarity index 99% rename from clinicadl/utils/network/vae/convolutional_VAE.py rename to clinicadl/network/vae/convolutional_VAE.py index 99c00a868..ab29c842e 100644 --- a/clinicadl/utils/network/vae/convolutional_VAE.py +++ b/clinicadl/network/vae/convolutional_VAE.py @@ -4,8 +4,8 @@ import torch.nn as nn import torch.nn.functional as F -from clinicadl.utils.network.network import Network -from clinicadl.utils.network.vae.vae_utils import multiply_list +from clinicadl.network.network import Network +from clinicadl.network.vae.vae_utils import multiply_list class CVAE_3D(Network): diff --git a/clinicadl/utils/network/vae/vae_layers.py b/clinicadl/network/vae/vae_layers.py similarity index 99% rename from clinicadl/utils/network/vae/vae_layers.py rename to clinicadl/network/vae/vae_layers.py index 3de57afc3..dfa9f0e15 100644 --- a/clinicadl/utils/network/vae/vae_layers.py +++ b/clinicadl/network/vae/vae_layers.py @@ -1,7 +1,7 @@ import torch.nn.functional as F from torch import nn -from clinicadl.utils.network.vae.vae_utils import get_norm2d, get_norm3d +from clinicadl.network.vae.vae_utils import get_norm2d, get_norm3d class EncoderLayer2D(nn.Module): diff --git a/clinicadl/utils/network/vae/vae_utils.py b/clinicadl/network/vae/vae_utils.py similarity index 100% rename from clinicadl/utils/network/vae/vae_utils.py rename to clinicadl/network/vae/vae_utils.py diff --git a/clinicadl/utils/network/vae/vanilla_vae.py b/clinicadl/network/vae/vanilla_vae.py similarity index 99% rename from clinicadl/utils/network/vae/vanilla_vae.py rename to clinicadl/network/vae/vanilla_vae.py index 1afe02329..200db6cc1 100644 --- a/clinicadl/utils/network/vae/vanilla_vae.py +++ b/clinicadl/network/vae/vanilla_vae.py @@ -1,8 +1,8 @@ import torch from torch import nn -from clinicadl.utils.network.vae.base_vae import BaseVAE -from clinicadl.utils.network.vae.vae_layers import ( +from clinicadl.network.vae.base_vae import BaseVAE +from clinicadl.network.vae.vae_layers import ( DecoderLayer3D, EncoderLayer3D, Flatten, diff --git a/clinicadl/config/config/optimization.py b/clinicadl/optimizer/optimization.py similarity index 100% rename from clinicadl/config/config/optimization.py rename to clinicadl/optimizer/optimization.py diff --git a/clinicadl/config/config/optimizer.py b/clinicadl/optimizer/optimizer.py similarity index 100% rename from clinicadl/config/config/optimizer.py rename to clinicadl/optimizer/optimizer.py diff --git a/clinicadl/predict/predict_manager.py b/clinicadl/predict/predict_manager.py index 2e570d058..6684cb149 100644 --- a/clinicadl/predict/predict_manager.py +++ b/clinicadl/predict/predict_manager.py @@ -13,6 +13,7 @@ from clinicadl.config.config.pipelines.interpret import InterpretConfig from clinicadl.config.config.pipelines.predict import PredictConfig +from clinicadl.transforms.transforms import get_transforms from clinicadl.utils.caps_dataset.data import ( return_dataset, ) @@ -23,7 +24,6 @@ ) from clinicadl.utils.maps_manager.ddp import DDP, cluster from clinicadl.utils.maps_manager.maps_manager import MapsManager -from clinicadl.utils.transforms.transforms import get_transforms logger = getLogger("clinicadl.predict_manager") level_list: List[str] = ["warning", "info", "debug"] @@ -887,7 +887,7 @@ def get_group_info( df = pd.read_csv(group_path / "data.tsv", sep="\t") json_path = group_path / "maps.json" - from clinicadl.utils.preprocessing import path_decoder + from clinicadl.preprocessing.preprocessing import path_decoder with json_path.open(mode="r") as f: parameters = json.load(f, object_hook=path_decoder) diff --git a/clinicadl/prepare_data/prepare_data.py b/clinicadl/prepare_data/prepare_data.py index 02a45ddbd..74f07f2f0 100644 --- a/clinicadl/prepare_data/prepare_data.py +++ b/clinicadl/prepare_data/prepare_data.py @@ -11,6 +11,7 @@ PrepareDataROIConfig, PrepareDataSliceConfig, ) +from clinicadl.preprocessing.preprocessing import write_preprocessing from clinicadl.utils.clinica_utils import ( check_caps_folder, clinicadl_file_reader, @@ -19,7 +20,6 @@ ) from clinicadl.utils.enum import ExtractionMethod, Pattern, Preprocessing, Template from clinicadl.utils.exceptions import ClinicaDLArgumentError -from clinicadl.utils.preprocessing import write_preprocessing from .prepare_data_utils import check_mask_list, compute_folder_and_file_type diff --git a/clinicadl/config/config/preprocessing.py b/clinicadl/preprocessing/config.py similarity index 100% rename from clinicadl/config/config/preprocessing.py rename to clinicadl/preprocessing/config.py diff --git a/clinicadl/utils/preprocessing.py b/clinicadl/preprocessing/preprocessing.py similarity index 100% rename from clinicadl/utils/preprocessing.py rename to clinicadl/preprocessing/preprocessing.py diff --git a/clinicadl/quality_check/pet_linear/utils.py b/clinicadl/quality_check/pet_linear/utils.py index 8f4d9abca..1edba9e15 100644 --- a/clinicadl/quality_check/pet_linear/utils.py +++ b/clinicadl/quality_check/pet_linear/utils.py @@ -6,7 +6,7 @@ import numpy as np -from clinicadl.utils.transforms.transforms import MinMaxNormalization +from clinicadl.transforms.transforms import MinMaxNormalization def get_metric(contour_np, image_np, inside): diff --git a/clinicadl/quality_check/t1_linear/utils.py b/clinicadl/quality_check/t1_linear/utils.py index b998f246b..2d08e08d4 100755 --- a/clinicadl/quality_check/t1_linear/utils.py +++ b/clinicadl/quality_check/t1_linear/utils.py @@ -30,7 +30,7 @@ def __init__( data_df (DataFrame): Subject and session list. """ - from clinicadl.utils.transforms.transforms import MinMaxNormalization + from clinicadl.transforms.transforms import MinMaxNormalization self.img_dir = img_dir self.df = data_df diff --git a/clinicadl/random_search/random_search_config.py b/clinicadl/random_search/random_search_config.py index cbf304c64..92f89b3b3 100644 --- a/clinicadl/random_search/random_search_config.py +++ b/clinicadl/random_search/random_search_config.py @@ -10,7 +10,7 @@ from clinicadl.config.config.pipelines.task.regression import ( RegressionConfig as BaseRegressionConfig, ) -from clinicadl.utils.config_utils import get_type_from_config_class as get_type +from clinicadl.config.config_utils import get_type_from_config_class as get_type from clinicadl.utils.enum import Normalization, Pooling, Task if TYPE_CHECKING: @@ -62,7 +62,7 @@ def to_tuple(cls, v): def training_config_for_random_models(base_training_config): base_model_config = get_type("model", base_training_config) - class ModelConfig(base_model_config): + class NetworkConfig(base_model_config): """Config class for random models.""" architecture: str = "RandomArchitecture" @@ -89,7 +89,7 @@ class TrainConfig(base_training_config): - n_fcblocks """ - model: ModelConfig + model: NetworkConfig return TrainConfig diff --git a/clinicadl/random_search/random_search_utils.py b/clinicadl/random_search/random_search_utils.py index f42a03181..a1036b5ff 100644 --- a/clinicadl/random_search/random_search_utils.py +++ b/clinicadl/random_search/random_search_utils.py @@ -4,10 +4,10 @@ import toml +from clinicadl.preprocessing.preprocessing import path_decoder, read_preprocessing from clinicadl.train.utils import extract_config_from_toml_file from clinicadl.utils.enum import Task from clinicadl.utils.exceptions import ClinicaDLConfigurationError -from clinicadl.utils.preprocessing import path_decoder, read_preprocessing def get_space_dict(launch_directory: Path) -> Dict[str, Any]: diff --git a/clinicadl/train/from_json/__init__.py b/clinicadl/train/from_json/__init__.py deleted file mode 100644 index 514d4a820..000000000 --- a/clinicadl/train/from_json/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .from_json_cli import cli diff --git a/clinicadl/train/list_models/__init__.py b/clinicadl/train/list_models/__init__.py deleted file mode 100644 index d1a7b1778..000000000 --- a/clinicadl/train/list_models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .list_models_cli import cli diff --git a/clinicadl/train/resume/resume.py b/clinicadl/train/resume.py similarity index 94% rename from clinicadl/train/resume/resume.py rename to clinicadl/train/resume.py index bc7158230..f80f5791b 100644 --- a/clinicadl/train/resume/resume.py +++ b/clinicadl/train/resume.py @@ -2,13 +2,13 @@ Automatic relaunch of jobs that were stopped before the end of training. Unfinished splits are detected as they do not contain a "performances" sub-folder """ - +# TODO: Remove this file and put everything in trainer.resume() ?? from logging import getLogger from pathlib import Path from clinicadl import MapsManager -from clinicadl.train.tasks import create_training_config -from clinicadl.train.trainer import Trainer +from clinicadl.train.tasks_utils import create_training_config +from clinicadl.trainer.trainer import Trainer def replace_arg(options, key_name, value): diff --git a/clinicadl/train/resume/__init__.py b/clinicadl/train/resume/__init__.py deleted file mode 100644 index e001d1f88..000000000 --- a/clinicadl/train/resume/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .resume_cli import cli diff --git a/clinicadl/train/tasks/__init__.py b/clinicadl/train/tasks/__init__.py deleted file mode 100644 index 7b9cd2ac8..000000000 --- a/clinicadl/train/tasks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .tasks_utils import create_training_config diff --git a/clinicadl/train/tasks/tasks_utils.py b/clinicadl/train/tasks_utils.py similarity index 95% rename from clinicadl/train/tasks/tasks_utils.py rename to clinicadl/train/tasks_utils.py index f05c3d5e5..307a6cc2f 100644 --- a/clinicadl/train/tasks/tasks_utils.py +++ b/clinicadl/train/tasks_utils.py @@ -1,3 +1,4 @@ +# TODO: to put in trainer ? trainer_utils.py ? from typing import Type, Union from clinicadl.config.config.pipelines.train import TrainConfig diff --git a/clinicadl/train/utils.py b/clinicadl/train/utils.py index 48713ecf9..f0919ddc2 100644 --- a/clinicadl/train/utils.py +++ b/clinicadl/train/utils.py @@ -1,3 +1,4 @@ +# TODO: merge with task_utils to create the trainer_utils ? from pathlib import Path from typing import Any, Dict @@ -5,10 +6,10 @@ import toml from click.core import ParameterSource +from clinicadl.preprocessing.preprocessing import path_decoder from clinicadl.utils.enum import Task from clinicadl.utils.exceptions import ClinicaDLConfigurationError from clinicadl.utils.maps_manager.maps_manager_utils import remove_unused_tasks -from clinicadl.utils.preprocessing import path_decoder def extract_config_from_toml_file(config_file: Path, task: Task) -> Dict[str, Any]: @@ -89,7 +90,7 @@ def get_model_list(architecture=None, input_size=None, model_layers=False): """ from inspect import getmembers, isclass - import clinicadl.utils.network as network_package + import clinicadl.network as network_package if not architecture: print("The list of currently available models is:") diff --git a/clinicadl/train/trainer/__init__.py b/clinicadl/trainer/__init__.py similarity index 100% rename from clinicadl/train/trainer/__init__.py rename to clinicadl/trainer/__init__.py diff --git a/clinicadl/train/trainer/trainer.py b/clinicadl/trainer/trainer.py similarity index 99% rename from clinicadl/train/trainer/trainer.py rename to clinicadl/trainer/trainer.py index 1fbcf9534..54d66ff54 100644 --- a/clinicadl/train/trainer/trainer.py +++ b/clinicadl/trainer/trainer.py @@ -21,7 +21,7 @@ from clinicadl.utils.maps_manager.logwriter import LogWriter from clinicadl.utils.metric_module import RetainBest from clinicadl.utils.seed import pl_worker_init_function, seed_everything -from clinicadl.utils.transforms.transforms import get_transforms +from clinicadl.transforms.transforms import get_transforms from clinicadl.utils.maps_manager import MapsManager from clinicadl.utils.seed import get_seed @@ -709,7 +709,7 @@ def _train( profiler = self._init_profiler() if self.config.callbacks.track_exp == "wandb": - from clinicadl.utils.tracking_exp import WandB_handler + from clinicadl.callbacks.tracking_exp import WandB_handler if self.config.lr_scheduler.adaptive_learning_rate: from torch.optim.lr_scheduler import ReduceLROnPlateau diff --git a/clinicadl/train/trainer/trainer_utils.py b/clinicadl/trainer/trainer_utils.py similarity index 100% rename from clinicadl/train/trainer/trainer_utils.py rename to clinicadl/trainer/trainer_utils.py diff --git a/clinicadl/config/config/transforms.py b/clinicadl/transforms/config.py similarity index 100% rename from clinicadl/config/config/transforms.py rename to clinicadl/transforms/config.py diff --git a/clinicadl/utils/transforms/transforms.py b/clinicadl/transforms/transforms.py similarity index 100% rename from clinicadl/utils/transforms/transforms.py rename to clinicadl/transforms/transforms.py diff --git a/clinicadl/tsvtools/analysis/analysis.py b/clinicadl/tsvtools/analysis/analysis.py index d089a05a0..c9b0a0732 100644 --- a/clinicadl/tsvtools/analysis/analysis.py +++ b/clinicadl/tsvtools/analysis/analysis.py @@ -7,8 +7,7 @@ import numpy as np import pandas as pd -from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError -from clinicadl.utils.tsvtools_utils import ( +from clinicadl.tsvtools.tsvtools_utils import ( add_demographics, cleaning_nan_diagnoses, find_label, @@ -16,6 +15,7 @@ merged_tsv_reader, next_session, ) +from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError logger = getLogger("clinicadl.tsvtools.analysis") diff --git a/clinicadl/tsvtools/get_labels/get_labels.py b/clinicadl/tsvtools/get_labels/get_labels.py index 629031f7c..041b301e9 100644 --- a/clinicadl/tsvtools/get_labels/get_labels.py +++ b/clinicadl/tsvtools/get_labels/get_labels.py @@ -19,15 +19,15 @@ import numpy as np import pandas as pd -from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError -from clinicadl.utils.maps_manager.iotools import commandline_to_json -from clinicadl.utils.tsvtools_utils import ( +from clinicadl.tsvtools.tsvtools_utils import ( cleaning_nan_diagnoses, find_label, first_session, last_session, neighbour_session, ) +from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError +from clinicadl.utils.maps_manager.iotools import commandline_to_json logger = getLogger("clinicadl.tsvtools") diff --git a/clinicadl/tsvtools/get_metadata/get_metadata.py b/clinicadl/tsvtools/get_metadata/get_metadata.py index d9376224a..5ba40a4f9 100644 --- a/clinicadl/tsvtools/get_metadata/get_metadata.py +++ b/clinicadl/tsvtools/get_metadata/get_metadata.py @@ -4,8 +4,8 @@ import numpy as np import pandas as pd +from clinicadl.tsvtools.tsvtools_utils import merged_tsv_reader from clinicadl.utils.exceptions import ClinicaDLArgumentError -from clinicadl.utils.tsvtools_utils import merged_tsv_reader logger = getLogger("clinicadl.tsvtools.get_metadata") diff --git a/clinicadl/tsvtools/get_progression/get_progression.py b/clinicadl/tsvtools/get_progression/get_progression.py index 4dd1b38e0..7376198d5 100644 --- a/clinicadl/tsvtools/get_progression/get_progression.py +++ b/clinicadl/tsvtools/get_progression/get_progression.py @@ -5,7 +5,7 @@ import pandas as pd from clinicadl.tsvtools.get_labels import infer_or_drop_diagnosis -from clinicadl.utils.tsvtools_utils import ( +from clinicadl.tsvtools.tsvtools_utils import ( after_end_screening, last_session, merged_tsv_reader, diff --git a/clinicadl/tsvtools/getlabels/getlabels.py b/clinicadl/tsvtools/getlabels/getlabels.py index 5bc6dc13c..a2f2d84d2 100644 --- a/clinicadl/tsvtools/getlabels/getlabels.py +++ b/clinicadl/tsvtools/getlabels/getlabels.py @@ -20,15 +20,15 @@ import numpy as np import pandas as pd -from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError -from clinicadl.utils.maps_manager.iotools import commandline_to_json -from clinicadl.utils.tsvtools_utils import ( +from clinicadl.tsvtools.tsvtools_utils import ( after_end_screening, find_label, first_session, last_session, neighbour_session, ) +from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError +from clinicadl.utils.maps_manager.iotools import commandline_to_json logger = getLogger("clinicadl") diff --git a/clinicadl/tsvtools/kfold/kfold.py b/clinicadl/tsvtools/kfold/kfold.py index e60615cf9..12ffddd4b 100644 --- a/clinicadl/tsvtools/kfold/kfold.py +++ b/clinicadl/tsvtools/kfold/kfold.py @@ -7,9 +7,9 @@ import pandas as pd from sklearn.model_selection import StratifiedKFold +from clinicadl.tsvtools.tsvtools_utils import extract_baseline, retrieve_longitudinal from clinicadl.utils.exceptions import ClinicaDLTSVError from clinicadl.utils.maps_manager.iotools import commandline_to_json -from clinicadl.utils.tsvtools_utils import extract_baseline, retrieve_longitudinal sex_dict = {"M": 0, "F": 1} logger = getLogger("clinicadl.tsvtools.kfold") diff --git a/clinicadl/tsvtools/split/split.py b/clinicadl/tsvtools/split/split.py index 082b9bd49..cef621714 100644 --- a/clinicadl/tsvtools/split/split.py +++ b/clinicadl/tsvtools/split/split.py @@ -8,9 +8,7 @@ from scipy.stats import ks_2samp, ttest_ind from sklearn.model_selection import StratifiedShuffleSplit -from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError -from clinicadl.utils.maps_manager.iotools import commandline_to_json -from clinicadl.utils.tsvtools_utils import ( +from clinicadl.tsvtools.tsvtools_utils import ( category_conversion, chi2, complementary_list, @@ -20,6 +18,8 @@ remove_unicity, retrieve_longitudinal, ) +from clinicadl.utils.exceptions import ClinicaDLArgumentError, ClinicaDLTSVError +from clinicadl.utils.maps_manager.iotools import commandline_to_json sex_dict = {"M": 0, "F": 1} logger = getLogger("clinicadl.tsvtools.split") diff --git a/clinicadl/utils/tsvtools_utils.py b/clinicadl/tsvtools/tsvtools_utils.py similarity index 100% rename from clinicadl/utils/tsvtools_utils.py rename to clinicadl/tsvtools/tsvtools_utils.py diff --git a/clinicadl/utils/caps_dataset/data.py b/clinicadl/utils/caps_dataset/data.py index 9498b2363..ffbeda5f9 100644 --- a/clinicadl/utils/caps_dataset/data.py +++ b/clinicadl/utils/caps_dataset/data.py @@ -1,5 +1,5 @@ # coding: utf8 - +# TODO: create a folder for generate/ prepare_data/ data to deal with capsDataset objects ? import abc from logging import getLogger from pathlib import Path diff --git a/clinicadl/utils/cli_param/argument.py b/clinicadl/utils/cli_param/argument.py index d029fd864..78e6e1e96 100644 --- a/clinicadl/utils/cli_param/argument.py +++ b/clinicadl/utils/cli_param/argument.py @@ -1,4 +1,5 @@ """Common CLI arguments used by ClinicaDL pipelines.""" +# TODO: Remove this file, everything is now in the cmdline folder from pathlib import Path diff --git a/clinicadl/utils/cli_param/option.py b/clinicadl/utils/cli_param/option.py index 5677dee1e..6ff86cda2 100644 --- a/clinicadl/utils/cli_param/option.py +++ b/clinicadl/utils/cli_param/option.py @@ -1,5 +1,5 @@ """Common CLI options used by Clinica pipelines.""" - +# TODO: Remove this file, everything is now in the cmdline folder from pathlib import Path import click diff --git a/clinicadl/utils/cli_param/option_group.py b/clinicadl/utils/cli_param/option_group.py index 3d4f31c75..a2686f9ec 100644 --- a/clinicadl/utils/cli_param/option_group.py +++ b/clinicadl/utils/cli_param/option_group.py @@ -1,3 +1,5 @@ +# TODO: Remove this file, everything is now in the cmdline folder + from click_option_group import OptionGroup computational_group = OptionGroup( diff --git a/clinicadl/utils/maps_manager/maps_manager.py b/clinicadl/utils/maps_manager/maps_manager.py index 7583b6648..e2dfbaa3b 100644 --- a/clinicadl/utils/maps_manager/maps_manager.py +++ b/clinicadl/utils/maps_manager/maps_manager.py @@ -10,6 +10,8 @@ import torch.distributed as dist from torch.cuda.amp import autocast +from clinicadl.preprocessing.preprocessing import path_encoder +from clinicadl.transforms.transforms import get_transforms from clinicadl.utils.caps_dataset.data import ( return_dataset, ) @@ -24,8 +26,6 @@ add_default_values, read_json, ) -from clinicadl.utils.preprocessing import path_encoder -from clinicadl.utils.transforms.transforms import get_transforms logger = getLogger("clinicadl.maps_manager") level_list: List[str] = ["warning", "info", "debug"] @@ -596,7 +596,7 @@ def _write_information(self): """ from datetime import datetime - import clinicadl.utils.network as network_package + import clinicadl.network as network_package model_class = getattr(network_package, self.architecture) args = list( @@ -829,7 +829,7 @@ def _init_model( gpu (bool): If given, a new value for the device of the model will be computed. network (int): Index of the network trained (used in multi-network setting only). """ - import clinicadl.utils.network as network_package + import clinicadl.network as network_package logger.debug(f"Initialization of model {self.architecture}") # or choose to implement a dictionary diff --git a/clinicadl/utils/maps_manager/maps_manager_utils.py b/clinicadl/utils/maps_manager/maps_manager_utils.py index 99747b35b..e711342e5 100644 --- a/clinicadl/utils/maps_manager/maps_manager_utils.py +++ b/clinicadl/utils/maps_manager/maps_manager_utils.py @@ -5,7 +5,7 @@ import toml from clinicadl.prepare_data.prepare_data_utils import compute_folder_and_file_type -from clinicadl.utils.preprocessing import path_decoder, path_encoder +from clinicadl.preprocessing.preprocessing import path_decoder, path_encoder def add_default_values(user_dict: Dict[str, Any]) -> Dict[str, Any]: @@ -59,7 +59,7 @@ def read_json(json_path: Path) -> Dict[str, Any]: ------- A dictionary of training parameters. """ - from clinicadl.utils.preprocessing import path_decoder + from clinicadl.preprocessing.preprocessing import path_decoder with json_path.open(mode="r") as f: parameters = json.load(f, object_hook=path_decoder) diff --git a/clinicadl/utils/task_manager/reconstruction.py b/clinicadl/utils/task_manager/reconstruction.py index 66905938b..0b04fd74e 100644 --- a/clinicadl/utils/task_manager/reconstruction.py +++ b/clinicadl/utils/task_manager/reconstruction.py @@ -166,15 +166,15 @@ def get_criterion(criterion=None): f"Reconstruction loss must be chosen in {compatible_losses}." ) if criterion == "VAEGaussianLoss": - from clinicadl.utils.network.vae.vae_utils import VAEGaussianLoss + from clinicadl.network.vae.vae_utils import VAEGaussianLoss return VAEGaussianLoss elif criterion == "VAEBernoulliLoss": - from clinicadl.utils.network.vae.vae_utils import VAEBernoulliLoss + from clinicadl.network.vae.vae_utils import VAEBernoulliLoss return VAEBernoulliLoss elif criterion == "VAEContinuousBernoulliLoss": - from clinicadl.utils.network.vae.vae_utils import VAEContinuousBernoulliLoss + from clinicadl.network.vae.vae_utils import VAEContinuousBernoulliLoss return VAEContinuousBernoulliLoss return getattr(nn, criterion)() diff --git a/clinicadl/utils/task_manager/task_manager.py b/clinicadl/utils/task_manager/task_manager.py index 027aaf05a..c83fb7a13 100644 --- a/clinicadl/utils/task_manager/task_manager.py +++ b/clinicadl/utils/task_manager/task_manager.py @@ -9,10 +9,10 @@ from torch.nn.modules.loss import _Loss from torch.utils.data import DataLoader, Sampler +from clinicadl.network.network import Network from clinicadl.utils.caps_dataset.data import CapsDataset from clinicadl.utils.maps_manager.ddp import cluster from clinicadl.utils.metric_module import MetricModule -from clinicadl.utils.network.network import Network # TODO: add function to check that the output size of the network corresponds to what is expected to