Skip to content

Commit

Permalink
Made resolve_dataset_pattern return just dict
Browse files Browse the repository at this point in the history
Signed-off-by: Elena Khaustova <[email protected]>
  • Loading branch information
ElenaKhaustova committed Sep 11, 2024
1 parent 0a6946a commit fee7bd6
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 35 deletions.
12 changes: 6 additions & 6 deletions kedro/framework/cli/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,12 @@ def list_datasets(metadata: ProjectMetadata, pipeline: str, env: str) -> None:
# resolve any factory datasets in the pipeline
factory_ds_by_type = defaultdict(list)

resolved_configs = data_catalog.config_resolver.resolve_dataset_pattern(
default_ds
)
for ds_name, ds_config in zip(default_ds, resolved_configs):
for ds_name in default_ds:
if data_catalog.config_resolver.match_pattern(ds_name):
factory_ds_by_type[ds_config.get("type", "DefaultDataset")].append( # type: ignore[attr-defined]
ds_config = data_catalog.config_resolver.resolve_dataset_pattern(
ds_name
)
factory_ds_by_type[ds_config.get("type", "DefaultDataset")].append(
ds_name
)

Expand Down Expand Up @@ -253,7 +253,7 @@ def resolve_patterns(metadata: ProjectMetadata, env: str) -> None:
ds_config = data_catalog.config_resolver.resolve_dataset_pattern(ds_name)

# Exclude MemoryDatasets not set in the catalog explicitly
if ds_config is not None:
if ds_config:
explicit_datasets[ds_name] = ds_config

secho(yaml.dump(explicit_datasets))
47 changes: 20 additions & 27 deletions kedro/io/catalog_config_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,36 +229,29 @@ def _resolve_config_credentials(

return resolved_configs

def resolve_dataset_pattern(
self, datasets: str | list[str]
) -> dict[str, Any] | list[dict[str, Any]]:
def resolve_dataset_pattern(self, ds_name: str) -> dict[str, Any]:
"""Resolve dataset patterns and return resolved configurations based on the existing patterns."""
datasets_lst = [datasets] if isinstance(datasets, str) else datasets
resolved_configs = []

for ds_name in datasets_lst:
matched_pattern = self.match_pattern(ds_name)
if matched_pattern and ds_name not in self._resolved_configs:
pattern_config = self._get_pattern_config(matched_pattern)
ds_config = self._resolve_dataset_config(
ds_name, matched_pattern, copy.deepcopy(pattern_config)
matched_pattern = self.match_pattern(ds_name)

if matched_pattern and ds_name not in self._resolved_configs:
pattern_config = self._get_pattern_config(matched_pattern)
ds_config = self._resolve_dataset_config(
ds_name, matched_pattern, copy.deepcopy(pattern_config)
)

if (
self._pattern_specificity(matched_pattern) == 0
and matched_pattern in self._default_pattern
):
self._logger.warning(
"Config from the dataset factory pattern '%s' in the catalog will be used to "
"override the default dataset creation for '%s'",
matched_pattern,
ds_name,
)
return ds_config

if (
self._pattern_specificity(matched_pattern) == 0
and matched_pattern in self._default_pattern
):
self._logger.warning(
"Config from the dataset factory pattern '%s' in the catalog will be used to "
"override the default dataset creation for '%s'",
matched_pattern,
ds_name,
)
resolved_configs.append(ds_config)
else:
resolved_configs.append(self._resolved_configs.get(ds_name, None))

return resolved_configs[0] if isinstance(datasets, str) else resolved_configs
return self._resolved_configs.get(ds_name, {})

def add_runtime_patterns(self, dataset_patterns: Patterns) -> None:
"""Add new runtime patterns and re-sort them."""
Expand Down
4 changes: 2 additions & 2 deletions kedro/io/data_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -326,10 +326,10 @@ def _get_dataset(
) -> AbstractDataset:
ds_config = self._config_resolver.resolve_dataset_pattern(dataset_name)

if dataset_name not in self._datasets and ds_config is not None:
if dataset_name not in self._datasets and ds_config:
ds = AbstractDataset.from_config(
dataset_name,
ds_config, # type: ignore[arg-type]
ds_config,
self._load_versions.get(dataset_name),
self._save_version,
)
Expand Down

0 comments on commit fee7bd6

Please sign in to comment.