Fix eval bug

Signed-off-by: heyufan1995 <[email protected]>
Project-MONAI · Sep 26, 2024 · 4988add · 4988add
1 parent 78f2902
commit 4988add
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 33 deletions.
diff --git a/models/vista3d/configs/data.yaml b/models/vista3d/configs/data.yaml
diff --git a/models/vista3d/configs/evaluate.json b/models/vista3d/configs/evaluate.json
@@ -3,7 +3,6 @@
     "dataset_dir": "/data/Task09_Spleen",
     "output_dir": "$@bundle_root + '/eval'",
     "ckpt_path": "$@bundle_root + '/models/model.pt'",
-    "val_dataset_cache_rate": 0.0,
     "patch_size": [
         128,
         128,
@@ -14,19 +13,16 @@
         1.5,
         1.5
     ],
-    "cache_cls_idx#activate": false,
     "label_mappings": {
         "default": [
             [
                 1,
-                25
+                3
             ]
         ]
     },
-    "label_set": "$[0] + list(x[1] for x in @label_mappings#default)",
-    "val_label_set": "$[0] + list(x[0] for x in @label_mappings#default)",
-    "num_classes": 118,
-    "output_classes": 118,
+    "label_set": "$list(x[1] for x in @label_mappings#default)",
+    "validate#evaluator#hyper_kwargs#val_head": "auto", 
     "validate#preprocessing": {
         "_target_": "Compose",
         "transforms": [
@@ -107,19 +103,14 @@
                 "device": "cpu",
                 "_disabled_": true
             },
-            {
-                "_target_": "Activationsd",
-                "keys": "pred",
-                "sigmoid": true
-            },
             {
                 "_target_": "monai.apps.vista3d.transforms.VistaPostTransformd",
                 "keys": "pred"
             },
             {
                 "_target_": "Invertd",
                 "keys": "pred",
-                "transform": "@validate#preprocessing",
+                "transform": "$copy.deepcopy(@validate#preprocessing)",
                 "orig_keys": "image",
                 "nearest_interp": true,
                 "to_tensor": true
@@ -134,6 +125,12 @@
                 "keys": "pred",
                 "resample": false,
                 "output_dir": "@output_dir"
+            },
+            {
+                "_target_": "monai.apps.vista3d.transforms.Relabeld",
+                "keys": ["pred", "label"],
+                "label_mappings": "${'default': [[c, i+1] for i, c in enumerate(@label_set)]}",
+                "dtype": "$torch.uint8"
             }
         ]
     },
@@ -161,15 +158,11 @@
             "metric_details": "*",
             "summary_ops": "*"
         }
-    ],
+    ], 
     "validate#dataset": {
-        "_target_": "CacheDataset",
-        "data": "$list(@val_datalist)+list(@train_datalist)",
-        "transform": "@validate#preprocessing",
-        "cache_rate": "@val_dataset_cache_rate",
-        "hash_as_key": true,
-        "num_workers": "@num_cache_workers",
-        "progress": "@show_cache_progress"
+        "_target_": "Dataset",
+        "data": "$list(@val_datalist)",
+        "transform": "@validate#preprocessing"
     },
     "run": [
         "$@validate#evaluator.run()"

diff --git a/models/vista3d/configs/train.json b/models/vista3d/configs/train.json
@@ -3,7 +3,8 @@
         "$import glob",
         "$import os",
         "$import scripts",
-        "$import ignite"
+        "$import ignite",
+        "$import copy"
     ],
     "bundle_root": ".",
     "ckpt_dir": "$@bundle_root + '/models'",

diff --git a/models/vista3d/docs/README.md b/models/vista3d/docs/README.md
@@ -132,21 +132,26 @@ torchrun --nnodes=1 --nproc_per_node=8 -m monai.bundle run \
 
 
 ## Evaluation
+Evaluation can be used to calculate dice scores for the model or finetuned model. Change `ckpt_path` to the checkpoint user wish to evaluate.
+The dice score is calculated on the original image spacing with invertd, while the dice score during finetuning is on resampled space. 
 
-`configs/data.yaml` shows potential configurations for each specific dataset for evaluation.
+```
+NOTE: Evaluation does not support point evaluation.`"validate#evaluator#hyper_kwargs#val_head` is always set to `auto`.  
+```
 
 Single-GPU:
 ```
 python -m monai.bundle run \
-	--config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json','configs/data.yaml']"
+	--config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json']"
 ```
 
 Multi-GPU:
 ```
 torchrun --nnodes=1 --nproc_per_node=8 -m monai.bundle run \
-	--config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json','configs/mgpu_evaluate.json','configs/data.yaml']"
+	--config_file="['configs/train.json','configs/train_continual.json','configs/evaluate.json','configs/mgpu_evaluate.json']"
 ```
-
+#### Other explanatory items
+The `label_mapping` in `evaluation.json` does not contain `0` because the postprocessing performs argmax (`VistaPostTransformd`) and `0` prediction will hurt performance. While in continuous learning, `0` is included for validation because no argmax is performed and the validation is on `channel-wise` (include_background=False). Meanwhile, `Relabeld` in `postprocessing` is needed to map `label` and `pred` back to sequential indexes like 0,1,2,3,4 for dice calculation, since they are not in one-hot format. Evaluation does not support `point` but finetuning does since it does not perform argmax.
 
 ## Inference:
 For inference, VISTA3d bundle requires at least one prompt for segmentation. It supports label prompt, which is the index of the class for automatic segmentation.