Add I2I notebooks

biapy/engine/check_configuration.py

@@ -397,8 +397,6 @@ def check_configuration(cfg, jobname, check_data_paths=True):
                 raise ValueError("'PROBLEM.IMAGE_TO_IMAGE.MULTIPLE_RAW_ONE_TARGET_LOADER' can only be used if 'DATA.VAL.IN_MEMORY' == 'False'")
             if cfg.DATA.TEST.IN_MEMORY:
                 raise ValueError("'PROBLEM.IMAGE_TO_IMAGE.MULTIPLE_RAW_ONE_TARGET_LOADER' can only be used if 'DATA.TEST.IN_MEMORY' == 'False'")
-        if cfg.PROBLEM.NDIM == '3D':
-            raise ValueError("3D workflow not available for 'IMAGE_TO_IMAGE' yet")
 
     if cfg.DATA.EXTRACT_RANDOM_PATCH and cfg.DATA.PROBABILITY_MAP:
         if cfg.DATA.W_FOREGROUND+cfg.DATA.W_BACKGROUND != 1:
@@ -641,7 +639,7 @@ def check_configuration(cfg, jobname, check_data_paths=True):
 
     if cfg.MODEL.SOURCE == "biapy":
         assert cfg.MODEL.LAST_ACTIVATION.lower() in ["relu", "tanh", "leaky_relu", "elu", "gelu", "silu", "sigmoid","softmax", "linear", "none"], \
-            "Get unknown activation key {}".format(activation)
+            "Get unknown activation key {}".format(cfg.MODEL.LAST_ACTIVATION.lower())
 
         if cfg.MODEL.UPSAMPLE_LAYER.lower() not in ["upsampling", "convtranspose"]:
             raise ValueError("cfg.MODEL.UPSAMPLE_LAYER' needs to be one between ['upsampling', 'convtranspose']. Provided {}"
@@ -659,7 +657,7 @@ def check_configuration(cfg, jobname, check_data_paths=True):
                 assert cfg.MODEL.UNET_SR_UPSAMPLE_POSITION in ["pre", "post"], "'MODEL.UNET_SR_UPSAMPLE_POSITION' not in ['pre', 'post']"
         elif cfg.PROBLEM.TYPE == 'IMAGE_TO_IMAGE':
             if model_arch not in ['edsr', 'rcan', 'dfcan', 'wdsr', 'unet', 'resunet', 'resunet++', 'seunet', 'attention_unet', 'unetr', 'multiresunet']:
-                raise ValueError("Architectures available for 2D 'SUPER_RESOLUTION' are: ['edsr', 'rcan', 'dfcan', 'wdsr', 'unet', 'resunet', 'resunet++', 'seunet', 'attention_unet', 'unetr', 'multiresunet']")
+                raise ValueError("Architectures available for 2D 'IMAGE_TO_IMAGE' are: ['edsr', 'rcan', 'dfcan', 'wdsr', 'unet', 'resunet', 'resunet++', 'seunet', 'attention_unet', 'unetr', 'multiresunet']")
         elif cfg.PROBLEM.TYPE == 'SELF_SUPERVISED':
             if model_arch not in ['unet', 'resunet', 'resunet++', 'attention_unet', 'multiresunet', 'seunet',  
                 'unetr', 'edsr', 'rcan', 'dfcan', 'wdsr', 'vit', 'mae']:

biapy/utils/scripts/run_checks.py

@@ -37,7 +37,7 @@
     "description": "3D Semantic seg. Lucchi++. attention_unet. Basic DA.",
     "yaml": "test_2.yaml",
     "internal_checks": [
-        {"pattern": "Test Foreground IoU (merge patches)", "gt": True, "value": 0.7},
+        {"pattern": "Test Foreground IoU (merge patches)", "gt": True, "value": 0.55},
     ]
 }
 
@@ -273,7 +273,7 @@
     "enable": False,
     "run_experiment": False,
     "jobname": "test22",
-    "description": "3D classification. DermaMNIST 3D data. preprocess: resize, Cross-val. Basic DA. ViT",
+    "description": "3D classification. DermaMNIST 3D data. preprocess: resize, Cross-val. Basic DA. simple_cnn",
     "yaml": "test22.yaml",
     "internal_checks": [
         {"type": "regular", "pattern": "Validation top-5-accuracy:", "gt": True, "value": 0.7},
@@ -284,7 +284,7 @@
     "enable": False,
     "run_experiment": False,
     "jobname": "test23",
-    "description": "2D image to image. DermaMNIST 3D data. preprocess: resize, Cross-val. Basic DA. ViT",
+    "description": "3D classification. DermaMNIST 3D data. preprocess: resize. Basic DA. simple_cnn",
     "yaml": "test23.yaml",
     "internal_checks": [
         {"type": "regular", "pattern": "Validation top-5-accuracy:", "gt": True, "value": 0.7},
@@ -334,7 +334,18 @@
     "yaml": "test_27.yaml",
     "internal_checks": [
         {"type": "DatasetMatching", "pattern": "DatasetMatching(criterion='iou', thresh=0.3,", "nApparition": 1, "metric": "f1", 
-            "gt": True, "value": 0.4},
+            "gt": True, "value": 0.3},
+    ]
+}
+
+all_test_info["Test28"] = {
+    "enable": True,
+    "run_experiment": True,
+    "jobname": "test28",
+    "description": "3D Image to image. Nuclear_Pore_complex_3D data. in memory true. val 0.1 of train.",
+    "yaml": "test_28.yaml",
+    "internal_checks": [
+        {"type": "regular", "pattern": "Validation PSNR:", "gt": True, "value": 20.0},
     ]
 }
 
@@ -496,7 +507,7 @@
 image_to_image_folder = os.path.join(data_folder, "image_to_image")
 
 # 2D
-image_to_image_2d_template = "https://raw.githubusercontent.com/BiaPyX/BiaPy/master/templates/image-to-image/lightmycells/lightmycells_actin.yaml"
+image_to_image_2d_template = "https://raw.githubusercontent.com/BiaPyX/BiaPy/master/templates/image-to-image/2d_image-to-image.yaml"
 image_to_image_2d_template_local = os.path.join(image_to_image_folder, "2d_image_to_image.yaml")
 image_to_image_2d_data_drive_link = "https://drive.google.com/uc?id=1L8AXNjh0_updVI3-v1duf6CbcZb8uZK7"
 image_to_image_2d_data_filename = "Dapi_dataset.zip"
@@ -505,7 +516,14 @@
 # 2D ligthmycells
 image_to_image_light_2d_data_drive_link = "https://drive.google.com/uc?id=1SU4u-bcM1ZaDzEYg-d8W3zP6Yq2o8eKV"
 image_to_image_light_2d_data_filename = "reduced_actin_lightmycells.zip"
-image_to_image_light_2d_data_outpath = os.path.join(classification_folder, "reduced_actin_lightmycells")
+image_to_image_light_2d_data_outpath = os.path.join(image_to_image_folder, "reduced_actin_lightmycells")
+
+# 3D
+image_to_image_3d_template = "https://raw.githubusercontent.com/BiaPyX/BiaPy/master/templates/image-to-image/3d_image-to-image.yaml"
+image_to_image_3d_template_local = os.path.join(image_to_image_folder, "3d_image_to_image.yaml")
+image_to_image_3d_data_drive_link = "https://drive.google.com/uc?id=1jL0bn2X3OFaV5T-6KR1g6fPDllH-LWzm"
+image_to_image_3d_data_filename = "Nuclear_Pore_complex_3D.zip"
+image_to_image_3d_data_outpath = os.path.join(image_to_image_folder, "Nuclear_Pore_complex_3D")
 
 if not os.path.exists(biapy_folder): 
     raise ValueError(f"BiaPy not found in: {biapy_folder}")
@@ -832,6 +850,21 @@
         print("Downloading image_to_image YAML . . .") 
         _, _ = urllib.request.urlretrieve(image_to_image_2d_template, filename=image_to_image_2d_template_local)
 
+# General things: 3D Data + YAML donwload
+if not os.path.exists(image_to_image_3d_data_outpath) and all_test_info["Test28"]["enable"]:
+    print("Downloading 3D image to image data . . .")
+
+    os.makedirs(image_to_image_folder, exist_ok=True)
+    os.chdir(image_to_image_folder)
+    gdown.download(image_to_image_3d_data_drive_link, image_to_image_3d_data_filename, quiet=True)
+
+    with ZipFile(os.path.join(image_to_image_folder, image_to_image_3d_data_filename), 'r') as zObject: 
+        zObject.extractall(path=image_to_image_3d_data_outpath) 
+
+    if not os.path.exists(image_to_image_3d_template_local):
+        print("Downloading image to image YAML . . .") 
+        _, _ = urllib.request.urlretrieve(image_to_image_3d_template, filename=image_to_image_3d_template_local)
+
 
 def print_inventory(dct): 
     for item, amount in dct.items():  # dct.iteritems() in Python 2
@@ -2632,12 +2665,8 @@ def runjob(test_info, results_folder, yaml_file, biapy_folder, multigpu=False):
     biapy_config['DATA']['TRAIN']['PATH'] = os.path.join(classification_3d_data_outpath, "data", "train")
     biapy_config['DATA']['TRAIN']['IN_MEMORY'] = True
     biapy_config['DATA']['VAL']['FROM_TRAIN'] = True
-    biapy_config['DATA']['VAL']['CROSS_VAL'] = True
-    biapy_config['DATA']['VAL']['CROSS_VAL_NFOLD'] = 5
-    biapy_config['DATA']['VAL']['CROSS_VAL_FOLD'] = 3
     biapy_config['DATA']['TEST']['PATH'] = os.path.join(classification_3d_data_outpath, "data", "test")
     biapy_config['DATA']['TEST']['IN_MEMORY'] = True
-    biapy_config['DATA']['TEST']['USE_VAL_AS_TEST'] = True
 
     biapy_config['TRAIN']['ENABLE'] = True
     biapy_config['TRAIN']['EPOCHS'] = 5
@@ -3050,4 +3079,73 @@ def runjob(test_info, results_folder, yaml_file, biapy_folder, multigpu=False):
     print_result(results, all_test_info["Test27"]["jobname"], int_checks)
 
 
+#~~~~~~~~~~~~
+# Test 28
+#~~~~~~~~~~~~
+if all_test_info["Test28"]["enable"]:
+    print("######")
+    print("Running Test 28")
+    print_inventory(all_test_info["Test28"])
+
+    #*******************
+    # File preparation 
+    #*******************
+    # Open config file
+    with open(image_to_image_3d_template_local, 'r') as stream:
+        try:
+            biapy_config = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            raise ValueError(exc)
+
+    biapy_config['DATA']['PATCH_SIZE'] = "(6,128,128,1)"
+    biapy_config['DATA']['TRAIN']['PATH'] = os.path.join(image_to_image_3d_data_outpath, "data", "train", "x")
+    biapy_config['DATA']['TRAIN']['GT_PATH'] = os.path.join(image_to_image_3d_data_outpath, "data", "train", "y")
+    biapy_config['DATA']['TRAIN']['IN_MEMORY'] = True
+    biapy_config['DATA']['VAL']['FROM_TRAIN'] = True
+    biapy_config['DATA']['VAL']['SPLIT_TRAIN'] = 0.1
+    biapy_config['DATA']['TEST']['PATH'] = os.path.join(image_to_image_3d_data_outpath, "data", "test", "x")
+    biapy_config['DATA']['TEST']['GT_PATH'] = os.path.join(image_to_image_3d_data_outpath, "data", "test", "y")
+    biapy_config['DATA']['TEST']['IN_MEMORY'] = False
+    biapy_config['DATA']['TEST']['PADDING'] = "(0,24,24)"
+
+    biapy_config['TRAIN']['ENABLE'] = True
+    biapy_config['TRAIN']['EPOCHS'] = 15
+    biapy_config['TRAIN']['PATIENCE'] = -1 
+
+    biapy_config['MODEL']['ARCHITECTURE'] = 'resunet'
+    biapy_config['MODEL']['Z_DOWN'] = [1,1,1,1]
+    biapy_config['MODEL']['LOAD_CHECKPOINT'] = False
+
+    biapy_config['TEST']['ENABLE'] = True
+
+    # Save file
+    test_file = os.path.join(image_to_image_folder, all_test_info["Test28"]["yaml"])
+    with open(test_file, 'w') as outfile:
+        yaml.dump(biapy_config, outfile, default_flow_style=False)
+
+    # Run  
+    if all_test_info["Test28"]["run_experiment"]:
+        runjob(all_test_info["Test28"], results_folder, test_file, biapy_folder)
+
+    # Check  
+    results = []
+    res, last_lines = check_finished(all_test_info["Test28"], "Test 28")
+    if not res:
+        print("Internal check not passed: seems that it didn't finish")
+    results.append(res)
+    int_checks = 1
+    for checks in all_test_info["Test28"]["internal_checks"]: 
+        if checks["type"] == "regular":
+            results.append(check_value(last_lines, checks["pattern"], checks["value"], checks["gt"]))
+        else:
+            results.append(check_DatasetMatching(last_lines, checks["pattern"], checks["value"], gt=checks["gt"], 
+                value_to_check=checks["nApparition"], metric=checks["metric"]))
+        int_checks += 1
+        if not results[-1]:
+            print("Internal check not passed: {} {} {}".format(checks["pattern"], checks["gt"], checks["value"]))
+
+    # Test result  
+    print_result(results, all_test_info["Test28"]["jobname"], int_checks)
+
+
 print("Finish tests!!")