diff --git a/monai/bundle/config_parser.py b/monai/bundle/config_parser.py index 9a9d274adf3..70993a71a5a 100644 --- a/monai/bundle/config_parser.py +++ b/monai/bundle/config_parser.py @@ -38,19 +38,17 @@ class ConfigParser: config = { "my_dims": 2, "dims_1": "$@my_dims + 1", - "my_xform": {"": "LoadImage"}, - "my_net": {"": "BasicUNet", - "": {"spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}}, - "trainer": {"": "SupervisedTrainer", - "": {"network": "@my_net", "preprocessing": "@my_xform"}} + "my_xform": {"_name_": "LoadImage"}, + "my_net": {"_name_": "BasicUNet", "spatial_dims": "@dims_1", "in_channels": 1, "out_channels": 4}, + "trainer": {"_name_": "SupervisedTrainer", "network": "@my_net", "preprocessing": "@my_xform"} } # in the example $@my_dims + 1 is an expression, which adds 1 to the value of @my_dims parser = ConfigParser(config) # get/set configuration content, the set method should happen before calling parse() - print(parser["my_net"][""]["in_channels"]) # original input channels 1 - parser["my_net"][""]["in_channels"] = 4 # change input channels to 4 - print(parser["my_net"][""]["in_channels"]) + print(parser["my_net"]["in_channels"]) # original input channels 1 + parser["my_net"]["in_channels"] = 4 # change input channels to 4 + print(parser["my_net"]["in_channels"]) # instantiate the network component parser.parse(True) @@ -107,7 +105,7 @@ def __getitem__(self, id: Union[str, int]): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. """ if id == "": @@ -129,7 +127,7 @@ def __setitem__(self, id: Union[str, int], config: Any): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. config: config to set at location ``id``. """ @@ -171,7 +169,7 @@ def _do_resolve(self, config: Any): """ Recursively resolve the config content to replace the macro tokens with target content. The macro tokens start with "%", can be from another structured file, like: - ``{"net": "%default_net"}``, ``{"net": "%/data/config.json#net#"}``. + ``{"net": "%default_net"}``, ``{"net": "%/data/config.json#net"}``. Args: config: input config file to resolve. @@ -190,7 +188,7 @@ def resolve_macro(self): """ Recursively resolve `self.config` to replace the macro tokens with target content. The macro tokens are marked as starting with "%", can be from another structured file, like: - ``"%default_net"``, ``"%/data/config.json#net#"``. + ``"%default_net"``, ``"%/data/config.json#net"``. """ self.set(self._do_resolve(config=deepcopy(self.get()))) @@ -204,7 +202,7 @@ def _do_parse(self, config, id: str = ""): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. """ if isinstance(config, (dict, list)): @@ -248,7 +246,7 @@ def get_parsed_content(self, id: str = "", **kwargs): id: id of the ``ConfigItem``, ``"#"`` in id are interpreted as special characters to go one level further into the nested structures. Use digits indexing from "0" for list or other strings for dict. - For example: ``"xform#5"``, ``"net##channels"``. ``""`` indicates the entire ``self.config``. + For example: ``"xform#5"``, ``"net#channels"``. ``""`` indicates the entire ``self.config``. kwargs: additional keyword arguments to be passed to ``_resolve_one_item``. Currently support ``reset`` (for parse), ``instantiate`` and ``eval_expr``. All defaulting to True. diff --git a/monai/bundle/config_reader.py b/monai/bundle/config_reader.py index 4be7bf0fa8c..170883c6162 100644 --- a/monai/bundle/config_reader.py +++ b/monai/bundle/config_reader.py @@ -38,7 +38,7 @@ class ConfigReader: suffixes = ("json", "yaml", "yml") suffix_match = rf"\.({'|'.join(suffixes)})" path_match = rf"(.*{suffix_match}$)" - meta_key = "" # field key to save metadata + meta_key = "_meta_" # field key to save metadata sep = "#" # separator for file path and the id of content in the file def __init__(self): @@ -122,7 +122,7 @@ def split_path_id(cls, src: str) -> Tuple[str, str]: def read_meta(self, f: Union[PathLike, Sequence[PathLike], Dict], **kwargs): """ Read the metadata from specified JSON or YAML file. - The metadata as a dictionary will be stored at ``self.config[""]``. + The metadata as a dictionary will be stored at ``self.config["_meta_"]``. Args: f: filepath of the metadata file, the content must be a dictionary, diff --git a/monai/bundle/reference_resolver.py b/monai/bundle/reference_resolver.py index b3834bf76c2..8a8b5ed89dc 100644 --- a/monai/bundle/reference_resolver.py +++ b/monai/bundle/reference_resolver.py @@ -47,8 +47,8 @@ class ReferenceResolver: _vars = "__local_refs" sep = "#" # separator for key indexing ref = "@" # reference prefix - # match a reference string, e.g. "@id#key", "@id#key#0", "@##key" - id_matcher = re.compile(rf"{ref}(?:(?:<\w*>)|(?:\w*))(?:(?:{sep}<\w*>)|(?:{sep}\w*))*") + # match a reference string, e.g. "@id#key", "@id#key#0", "@_name_#key" + id_matcher = re.compile(rf"{ref}(?:\w*)(?:{sep}\w*)*") def __init__(self, items: Optional[Sequence[ConfigItem]] = None): # save the items in a dictionary with the `ConfigItem.id` as key diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py index 820726fb46c..a5749c1e1e3 100644 --- a/monai/bundle/scripts.py +++ b/monai/bundle/scripts.py @@ -34,13 +34,13 @@ def run( python -m monai.bundle run --meta_file --config_file --target_id trainer # Override config values at runtime by specifying the component id and its new value: - python -m monai.bundle run --"net##input_chns" 1 ... + python -m monai.bundle run --net#input_chns 1 ... # Override config values with another config file `/path/to/another.json`: - python -m monai.bundle run --"net#" "%/path/to/another.json" ... + python -m monai.bundle run --net %/path/to/another.json ... # Override config values with part content of another config file: - python -m monai.bundle run --"net#" "%/data/other.json#net_arg" ... + python -m monai.bundle run --net %/data/other.json#net_arg ... # Set default args of `run` in a JSON / YAML file, help to record and simplify the command line. # Other args still can override the default args at runtime: @@ -55,7 +55,7 @@ def run( args_file: a JSON or YAML file to provide default values for `meta_file`, `config_file`, `target_id` and override pairs. so that the command line inputs can be simplified. override: id-value pairs to override or add the corresponding config content. - e.g. ``--"net##input_chns" 42``. + e.g. ``--net#input_chns 42``. """ k_v = zip(["meta_file", "config_file", "target_id"], [meta_file, config_file, target_id]) diff --git a/tests/test_bundle_run.py b/tests/test_bundle_run.py index f25c4e256e0..d6c41271f5e 100644 --- a/tests/test_bundle_run.py +++ b/tests/test_bundle_run.py @@ -60,12 +60,12 @@ def test_shape(self, config_file, expected_shape): saver = LoadImage(image_only=True) if sys.platform == "win32": - override = "--network $@network_def.to(@device) --dataset# Dataset" + override = "--network $@network_def.to(@device) --dataset#_name_ Dataset" else: - override = f"--network %{overridefile1}#move_net --dataset# %{overridefile2}" + override = f"--network %{overridefile1}#move_net --dataset#_name_ %{overridefile2}" # test with `monai.bundle` as CLI entry directly cmd = "-m monai.bundle run --target_id evaluator" - cmd += f" --postprocessing##transforms#2##output_postfix seg {override}" + cmd += f" --postprocessing#transforms#2#output_postfix seg {override}" la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] ret = subprocess.check_call(la + ["--args_file", def_args_file]) self.assertEqual(ret, 0) @@ -73,7 +73,7 @@ def test_shape(self, config_file, expected_shape): # here test the script with `google fire` tool as CLI cmd = "-m fire monai.bundle.scripts run --target_id evaluator" - cmd += f" --evaluator##amp False {override}" + cmd += f" --evaluator#amp False {override}" la = [f"{sys.executable}"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file] ret = subprocess.check_call(la) self.assertEqual(ret, 0) diff --git a/tests/test_config_item.py b/tests/test_config_item.py index 9ce08561f3b..507b5a7d928 100644 --- a/tests/test_config_item.py +++ b/tests/test_config_item.py @@ -34,10 +34,7 @@ # test `_disabled_` with string TEST_CASE_5 = [{"_name_": "LoadImaged", "_disabled_": "true", "keys": ["image"]}, dict] # test non-monai modules and excludes -TEST_CASE_6 = [ - {"_path_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, - torch.optim.Adam, -] +TEST_CASE_6 = [{"_path_": "torch.optim.Adam", "params": torch.nn.PReLU().parameters(), "lr": 1e-4}, torch.optim.Adam] TEST_CASE_7 = [{"_name_": "decollate_batch", "detach": True, "pad": True}, partial] # test args contains "name" field TEST_CASE_8 = [ diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 5b5aa2b816a..c57ca48cbd8 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -25,24 +25,21 @@ TEST_CASE_1 = [ { "transform": { - "": "Compose", - "": { - "transforms": [ - {"": "LoadImaged", "": {"keys": "image"}}, - { - "": "RandTorchVisiond", - "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, - }, - ] - }, + "_name_": "Compose", + "transforms": [ + {"_name_": "LoadImaged", "keys": "image"}, + {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, + ], }, - "dataset": {"": "Dataset", "": {"data": [1, 2], "transform": "@transform"}}, + "dataset": {"_name_": "Dataset", "data": [1, 2], "transform": "@transform"}, "dataloader": { - "": "DataLoader", - "": {"dataset": "@dataset", "batch_size": 2, "collate_fn": "monai.data.list_data_collate"}, + "_name_": "DataLoader", + "dataset": "@dataset", + "batch_size": 2, + "collate_fn": "monai.data.list_data_collate", }, }, - ["transform", "transform##transforms#0", "transform##transforms#1", "dataset", "dataloader"], + ["transform", "transform#transforms#0", "transform#transforms#1", "dataset", "dataloader"], [Compose, LoadImaged, RandTorchVisiond, Dataset, DataLoader], ] @@ -67,9 +64,9 @@ def __call__(self, a, b): "cls_func": "$TestClass.cls_compute", "lambda_static_func": "$lambda x, y: TestClass.compute(x, y)", "lambda_cls_func": "$lambda x, y: TestClass.cls_compute(x, y)", - "compute": {"": "tests.test_config_parser.TestClass.compute", "": {"func": "@basic_func"}}, - "cls_compute": {"": "tests.test_config_parser.TestClass.cls_compute", "": {"func": "@basic_func"}}, - "call_compute": {"": "tests.test_config_parser.TestClass"}, + "compute": {"_path_": "tests.test_config_parser.TestClass.compute", "func": "@basic_func"}, + "cls_compute": {"_path_": "tests.test_config_parser.TestClass.cls_compute", "func": "@basic_func"}, + "call_compute": {"_path_": "tests.test_config_parser.TestClass"}, "error_func": "$TestClass.__call__", "": "$lambda x, y: x + y", } @@ -78,17 +75,17 @@ def __call__(self, a, b): class TestConfigComponent(unittest.TestCase): def test_config_content(self): - test_config = {"preprocessing": [{"": "LoadImage"}], "dataset": {"": "Dataset"}} + test_config = {"preprocessing": [{"_name_": "LoadImage"}], "dataset": {"_name_": "Dataset"}} parser = ConfigParser(config=test_config) # test `get`, `set`, `__getitem__`, `__setitem__` self.assertEqual(str(parser.get()), str(test_config)) parser.set(config=test_config) self.assertListEqual(parser["preprocessing"], test_config["preprocessing"]) - parser["dataset"] = {"": "CacheDataset"} - self.assertEqual(parser["dataset"][""], "CacheDataset") + parser["dataset"] = {"_name_": "CacheDataset"} + self.assertEqual(parser["dataset"]["_name_"], "CacheDataset") # test nested ids - parser["dataset#"] = "Dataset" - self.assertEqual(parser["dataset#"], "Dataset") + parser["dataset#_name_"] = "Dataset" + self.assertEqual(parser["dataset#_name_"], "Dataset") # test int id parser.set(["test1", "test2", "test3"]) parser[1] = "test4" @@ -99,11 +96,11 @@ def test_config_content(self): def test_parse(self, config, expected_ids, output_types): parser = ConfigParser(config=config, globals={"monai": "monai"}) # test lazy instantiation with original config content - parser["transform"][""]["transforms"][0][""]["keys"] = "label1" - self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label1") + parser["transform"]["transforms"][0]["keys"] = "label1" + self.assertEqual(parser.get_parsed_content(id="transform#transforms#0").keys[0], "label1") # test nested id - parser["transform##transforms#0##keys"] = "label2" - self.assertEqual(parser.get_parsed_content(id="transform##transforms#0").keys[0], "label2") + parser["transform#transforms#0#keys"] = "label2" + self.assertEqual(parser.get_parsed_content(id="transform#transforms#0").keys[0], "label2") for id, cls in zip(expected_ids, output_types): self.assertTrue(isinstance(parser.get_parsed_content(id), cls)) # test root content diff --git a/tests/test_reference_resolver.py b/tests/test_reference_resolver.py index e16a795c40c..66b86554026 100644 --- a/tests/test_reference_resolver.py +++ b/tests/test_reference_resolver.py @@ -27,11 +27,10 @@ TEST_CASE_1 = [ { # all the recursively parsed config items - "transform#1": {"": "LoadImaged", "": {"keys": ["image"]}}, - "transform#1#": "LoadImaged", - "transform#1#": {"keys": ["image"]}, - "transform#1##keys": ["image"], - "transform#1##keys#0": "image", + "transform#1": {"_name_": "LoadImaged", "keys": ["image"]}, + "transform#1#_name_": "LoadImaged", + "transform#1#keys": ["image"], + "transform#1#keys#0": "image", }, "transform#1", LoadImaged, @@ -40,20 +39,15 @@ TEST_CASE_2 = [ { # some the recursively parsed config items - "dataloader": { - "": "DataLoader", - "": {"dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, - }, - "dataset": {"": "Dataset", "": {"data": [1, 2]}}, - "dataloader#": "DataLoader", - "dataloader#": {"dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, - "dataloader##dataset": "@dataset", - "dataloader##collate_fn": "$monai.data.list_data_collate", - "dataset#": "Dataset", - "dataset#": {"data": [1, 2]}, - "dataset##data": [1, 2], - "dataset##data#0": 1, - "dataset##data#1": 2, + "dataloader": {"_name_": "DataLoader", "dataset": "@dataset", "collate_fn": "$monai.data.list_data_collate"}, + "dataset": {"_name_": "Dataset", "data": [1, 2]}, + "dataloader#_name_": "DataLoader", + "dataloader#dataset": "@dataset", + "dataloader#collate_fn": "$monai.data.list_data_collate", + "dataset#_name_": "Dataset", + "dataset#data": [1, 2], + "dataset#data#0": 1, + "dataset#data#1": 2, }, "dataloader", DataLoader, @@ -62,15 +56,11 @@ TEST_CASE_3 = [ { # all the recursively parsed config items - "transform#1": { - "": "RandTorchVisiond", - "": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, - }, - "transform#1#": "RandTorchVisiond", - "transform#1#": {"keys": "image", "name": "ColorJitter", "brightness": 0.25}, - "transform#1##keys": "image", - "transform#1##name": "ColorJitter", - "transform#1##brightness": 0.25, + "transform#1": {"_name_": "RandTorchVisiond", "keys": "image", "name": "ColorJitter", "brightness": 0.25}, + "transform#1#_name_": "RandTorchVisiond", + "transform#1#keys": "image", + "transform#1#name": "ColorJitter", + "transform#1#brightness": 0.25, }, "transform#1", RandTorchVisiond, @@ -97,7 +87,7 @@ def test_resolve(self, configs, expected_id, output_type): # test lazy instantiation item = resolver.get_item(expected_id, resolve=True) config = item.get_config() - config[""] = False + config["_disabled_"] = False item.update_config(config=config) if isinstance(item, ConfigComponent): result = item.instantiate() diff --git a/tests/testing_data/inference.json b/tests/testing_data/inference.json index 0a1cc0277e4..dcc64817d6e 100644 --- a/tests/testing_data/inference.json +++ b/tests/testing_data/inference.json @@ -1,126 +1,98 @@ { "device": "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')", "network_def": { - "": "UNet", - "": { - "spatial_dims": 3, - "in_channels": 1, - "out_channels": 2, - "channels": [ - 16, - 32, - 64, - 128, - 256 - ], - "strides": [ - 2, - 2, - 2, - 2 - ], - "num_res_units": 2, - "norm": "batch" - } + "_name_": "UNet", + "spatial_dims": 3, + "in_channels": 1, + "out_channels": 2, + "channels": [ + 16, + 32, + 64, + 128, + 256 + ], + "strides": [ + 2, + 2, + 2, + 2 + ], + "num_res_units": 2, + "norm": "batch" }, "network": "need override", "preprocessing": { - "": "Compose", - "": { - "transforms": [ - { - "": "LoadImaged", - "": { - "keys": "image" - } - }, - { - "": "EnsureChannelFirstd", - "": { - "keys": "image" - } - }, - { - "": "ScaleIntensityd", - "": { - "keys": "image" - } - }, - { - "": "EnsureTyped", - "": { - "keys": "image" - } - } - ] - } + "_name_": "Compose", + "transforms": [ + { + "_name_": "LoadImaged", + "keys": "image" + }, + { + "_name_": "EnsureChannelFirstd", + "keys": "image" + }, + { + "_name_": "ScaleIntensityd", + "keys": "image" + }, + { + "_name_": "EnsureTyped", + "keys": "image" + } + ] }, "dataset": { - "": "need override", - "": { - "data": "@#datalist", - "transform": "@preprocessing" - } + "_name_": "need override", + "data": "@_meta_#datalist", + "transform": "@preprocessing" }, "dataloader": { - "": "DataLoader", - "": { - "dataset": "@dataset", - "batch_size": 1, - "shuffle": false, - "num_workers": 4 - } + "_name_": "DataLoader", + "dataset": "@dataset", + "batch_size": 1, + "shuffle": false, + "num_workers": 4 }, "inferer": { - "": "SlidingWindowInferer", - "": { - "roi_size": [ - 96, - 96, - 96 - ], - "sw_batch_size": 4, - "overlap": 0.5 - } + "_name_": "SlidingWindowInferer", + "roi_size": [ + 96, + 96, + 96 + ], + "sw_batch_size": 4, + "overlap": 0.5 }, "postprocessing": { - "": "Compose", - "": { - "transforms": [ - { - "": "Activationsd", - "": { - "keys": "pred", - "softmax": true - } - }, - { - "": "AsDiscreted", - "": { - "keys": "pred", - "argmax": true - } - }, - { - "": "SaveImaged", - "": { - "keys": "pred", - "meta_keys": "image_meta_dict", - "output_dir": "@#output_dir" - } - } - ] - } + "_name_": "Compose", + "transforms": [ + { + "_name_": "Activationsd", + "keys": "pred", + "softmax": true + }, + { + "_name_": "AsDiscreted", + "keys": "pred", + "argmax": true + }, + { + "_name_": "SaveImaged", + "keys": "pred", + "meta_keys": "image_meta_dict", + "output_dir": "@_meta_#output_dir" + } + ] }, "evaluator": { - "": "SupervisedEvaluator", - "": { - "device": "@device", - "val_data_loader": "@dataloader", - "network": "@network", - "inferer": "@inferer", - "postprocessing": "@postprocessing", - "amp": false - } + "_name_": "SupervisedEvaluator", + "device": "@device", + "val_data_loader": "@dataloader", + "network": "@network", + "inferer": "@inferer", + "postprocessing": "@postprocessing", + "amp": false } } diff --git a/tests/testing_data/inference.yaml b/tests/testing_data/inference.yaml index 1a2f208a765..35876f70ff5 100644 --- a/tests/testing_data/inference.yaml +++ b/tests/testing_data/inference.yaml @@ -1,85 +1,71 @@ --- device: "$torch.device('cuda' if torch.cuda.is_available() else 'cpu')" network_def: - "": UNet - "": - spatial_dims: 3 - in_channels: 1 - out_channels: 2 - channels: - - 16 - - 32 - - 64 - - 128 - - 256 - strides: - - 2 - - 2 - - 2 - - 2 - num_res_units: 2 - norm: batch + _name_: UNet + spatial_dims: 3 + in_channels: 1 + out_channels: 2 + channels: + - 16 + - 32 + - 64 + - 128 + - 256 + strides: + - 2 + - 2 + - 2 + - 2 + num_res_units: 2 + norm: batch network: need override preprocessing: - "": Compose - "": - transforms: - - "": LoadImaged - "": - keys: image - - "": EnsureChannelFirstd - "": - keys: image - - "": ScaleIntensityd - "": - keys: image - - "": EnsureTyped - "": - keys: image + _name_: Compose + transforms: + - _name_: LoadImaged + keys: image + - _name_: EnsureChannelFirstd + keys: image + - _name_: ScaleIntensityd + keys: image + - _name_: EnsureTyped + keys: image dataset: - "": need override - "": - data: "@#datalist" - transform: "@preprocessing" + _name_: need override + data: "@_meta_#datalist" + transform: "@preprocessing" dataloader: - "": DataLoader - "": - dataset: "@dataset" - batch_size: 1 - shuffle: false - num_workers: 4 + _name_: DataLoader + dataset: "@dataset" + batch_size: 1 + shuffle: false + num_workers: 4 inferer: - "": SlidingWindowInferer - "": - roi_size: - - 96 - - 96 - - 96 - sw_batch_size: 4 - overlap: 0.5 + _name_: SlidingWindowInferer + roi_size: + - 96 + - 96 + - 96 + sw_batch_size: 4 + overlap: 0.5 postprocessing: - "": Compose - "": - transforms: - - "": Activationsd - "": - keys: pred - softmax: true - - "": AsDiscreted - "": - keys: pred - argmax: true - - "": SaveImaged - "": - keys: pred - meta_keys: image_meta_dict - output_dir: "@#output_dir" + _name_: Compose + transforms: + - _name_: Activationsd + keys: pred + softmax: true + - _name_: AsDiscreted + keys: pred + argmax: true + - _name_: SaveImaged + keys: pred + meta_keys: image_meta_dict + output_dir: "@_meta_#output_dir" evaluator: - "": SupervisedEvaluator - "": - device: "@device" - val_data_loader: "@dataloader" - network: "@network" - inferer: "@inferer" - postprocessing: "@postprocessing" - amp: false + _name_: SupervisedEvaluator + device: "@device" + val_data_loader: "@dataloader" + network: "@network" + inferer: "@inferer" + postprocessing: "@postprocessing" + amp: false