From 4988add2543474826276791799617c663844db3b Mon Sep 17 00:00:00 2001 From: heyufan1995 Date: Thu, 26 Sep 2024 13:11:24 -0400 Subject: [PATCH] Fix eval bug Signed-off-by: heyufan1995 --- models/vista3d/configs/data.yaml | 7 ------ models/vista3d/configs/evaluate.json | 35 +++++++++++----------------- models/vista3d/configs/train.json | 3 ++- models/vista3d/docs/README.md | 13 +++++++---- 4 files changed, 25 insertions(+), 33 deletions(-) delete mode 100644 models/vista3d/configs/data.yaml diff --git a/models/vista3d/configs/data.yaml b/models/vista3d/configs/data.yaml deleted file mode 100644 index d4b3ea89..00000000 --- a/models/vista3d/configs/data.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# this file shows specific settings for evaluation on each dataset -validate#postprocessing#transforms#0#_disabled_: false -eval_folder: "/eval/" -ckpt_path: "$@bundle_root + '/models/model.pt'" -label_set: "$list(set([i+1 for i in range(117)]) - set([22, 23, 15, 25, 19, 2, 26, 27, 28, 29, 117]))" -val_label_set: "$list(range(118))" -label_mappings: "${}" diff --git a/models/vista3d/configs/evaluate.json b/models/vista3d/configs/evaluate.json index 5615de23..c8b2db40 100644 --- a/models/vista3d/configs/evaluate.json +++ b/models/vista3d/configs/evaluate.json @@ -3,7 +3,6 @@ "dataset_dir": "/data/Task09_Spleen", "output_dir": "$@bundle_root + '/eval'", "ckpt_path": "$@bundle_root + '/models/model.pt'", - "val_dataset_cache_rate": 0.0, "patch_size": [ 128, 128, @@ -14,19 +13,16 @@ 1.5, 1.5 ], - "cache_cls_idx#activate": false, "label_mappings": { "default": [ [ 1, - 25 + 3 ] ] }, - "label_set": "$[0] + list(x[1] for x in @label_mappings#default)", - "val_label_set": "$[0] + list(x[0] for x in @label_mappings#default)", - "num_classes": 118, - "output_classes": 118, + "label_set": "$list(x[1] for x in @label_mappings#default)", + "validate#evaluator#hyper_kwargs#val_head": "auto", "validate#preprocessing": { "_target_": "Compose", "transforms": [ @@ -107,11 +103,6 @@ "device": "cpu", "_disabled_": true }, - { - "_target_": "Activationsd", - "keys": "pred", - "sigmoid": true - }, { "_target_": "monai.apps.vista3d.transforms.VistaPostTransformd", "keys": "pred" @@ -119,7 +110,7 @@ { "_target_": "Invertd", "keys": "pred", - "transform": "@validate#preprocessing", + "transform": "$copy.deepcopy(@validate#preprocessing)", "orig_keys": "image", "nearest_interp": true, "to_tensor": true @@ -134,6 +125,12 @@ "keys": "pred", "resample": false, "output_dir": "@output_dir" + }, + { + "_target_": "monai.apps.vista3d.transforms.Relabeld", + "keys": ["pred", "label"], + "label_mappings": "${'default': [[c, i+1] for i, c in enumerate(@label_set)]}", + "dtype": "$torch.uint8" } ] }, @@ -161,15 +158,11 @@ "metric_details": "*", "summary_ops": "*" } - ], + ], "validate#dataset": { - "_target_": "CacheDataset", - "data": "$list(@val_datalist)+list(@train_datalist)", - "transform": "@validate#preprocessing", - "cache_rate": "@val_dataset_cache_rate", - "hash_as_key": true, - "num_workers": "@num_cache_workers", - "progress": "@show_cache_progress" + "_target_": "Dataset", + "data": "$list(@val_datalist)", + "transform": "@validate#preprocessing" }, "run": [ "$@validate#evaluator.run()" diff --git a/models/vista3d/configs/train.json b/models/vista3d/configs/train.json index 614618e1..9af1647c 100644 --- a/models/vista3d/configs/train.json +++ b/models/vista3d/configs/train.json @@ -3,7 +3,8 @@ "$import glob", "$import os", "$import scripts", - "$import ignite" + "$import ignite", + "$import copy" ], "bundle_root": ".", "ckpt_dir": "$@bundle_root + '/models'", diff --git a/models/vista3d/docs/README.md b/models/vista3d/docs/README.md index 612571e1..b6b7cd3f 100644 --- a/models/vista3d/docs/README.md +++ b/models/vista3d/docs/README.md @@ -132,21 +132,26 @@ torchrun --nnodes=1 --nproc_per_node=8 -m monai.bundle run \ ## Evaluation +Evaluation can be used to calculate dice scores for the model or finetuned model. Change `ckpt_path` to the checkpoint user wish to evaluate. +The dice score is calculated on the original image spacing with invertd, while the dice score during finetuning is on resampled space. -`configs/data.yaml` shows potential configurations for each specific dataset for evaluation. +``` +NOTE: Evaluation does not support point evaluation.`"validate#evaluator#hyper_kwargs#val_head` is always set to `auto`. +``` Single-GPU: ``` python -m monai.bundle run \ - --config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json','configs/data.yaml']" + --config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json']" ``` Multi-GPU: ``` torchrun --nnodes=1 --nproc_per_node=8 -m monai.bundle run \ - --config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json','configs/mgpu_evaluate.json','configs/data.yaml']" + --config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json','configs/mgpu_evaluate.json']" ``` - +#### Other explanatory items +The `label_mapping` in `evaluation.json` does not contain `0` because the postprocessing performs argmax (`VistaPostTransformd`) and `0` prediction will hurt performance. While in continuous learning, `0` is included for validation because no argmax is performed and the validation is on `channel-wise` (include_background=False). Meanwhile, `Relabeld` in `postprocessing` is needed to map `label` and `pred` back to sequential indexes like 0,1,2,3,4 for dice calculation, since they are not in one-hot format. Evaluation does not support `point` but finetuning does since it does not perform argmax. ## Inference: For inference, VISTA3d bundle requires at least one prompt for segmentation. It supports label prompt, which is the index of the class for automatic segmentation.