From cc449266e7fe0e97f23e61b3c732b75a0d0a8dec Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 14 Feb 2024 09:11:13 +0100 Subject: [PATCH] Fix environment variable parsing (#423) * Fix environment variable parsing ```bash export PYICEBERG_CATALOG__SOMETHING__S3__REGION=eu-north-1 ``` Before: ```python >>> from pyiceberg.catalog import load_catalog >>> load_catalog('something').properties {'s3': {'region': 'eu-north-1'}, ...} ``` After: ```python >>> from pyiceberg.catalog import load_catalog >>> load_catalog('something').properties {'s3.region': 'eu-north-1', ...} ``` Which correspondents with the key `s3.region` that we use. * Add second test Co-authored-by: Hussein Awala * lint --------- Co-authored-by: Hussein Awala --- pyiceberg/utils/config.py | 4 ++-- tests/utils/test_config.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pyiceberg/utils/config.py b/pyiceberg/utils/config.py index 31ba0b36ed..e038005469 100644 --- a/pyiceberg/utils/config.py +++ b/pyiceberg/utils/config.py @@ -125,8 +125,8 @@ def set_property(_config: RecursiveDict, path: List[str], config_value: str) -> env_var_lower = env_var.lower() if env_var_lower.startswith(PYICEBERG.lower()): key = env_var_lower[len(PYICEBERG) :] - parts = key.split("__") - parts_normalized = [part.replace("_", "-") for part in parts] + parts = key.split("__", maxsplit=2) + parts_normalized = [part.replace('__', '.').replace("_", "-") for part in parts] set_property(config, parts_normalized, config_value) return config diff --git a/tests/utils/test_config.py b/tests/utils/test_config.py index d694e15562..5e3f72ccc6 100644 --- a/tests/utils/test_config.py +++ b/tests/utils/test_config.py @@ -41,6 +41,20 @@ def test_from_environment_variables_uppercase() -> None: assert Config().get_catalog_config("PRODUCTION") == {"uri": "https://service.io/api"} +@mock.patch.dict( + os.environ, + { + "PYICEBERG_CATALOG__PRODUCTION__S3__REGION": "eu-north-1", + "PYICEBERG_CATALOG__PRODUCTION__S3__ACCESS_KEY_ID": "username", + }, +) +def test_fix_nested_objects_from_environment_variables() -> None: + assert Config().get_catalog_config("PRODUCTION") == { + 's3.region': 'eu-north-1', + 's3.access-key-id': 'username', + } + + def test_from_configuration_files(tmp_path_factory: pytest.TempPathFactory) -> None: config_path = str(tmp_path_factory.mktemp("config")) with open(f"{config_path}/.pyiceberg.yaml", "w", encoding=UTF8) as file: