From cbbe6e5987300de0ee94c2d201542943ecbcdb85 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Mon, 26 Jun 2023 10:37:21 -0700 Subject: [PATCH 1/3] [Index configuration tool] Add support for insecure HTTPS endpoint This commit adds support for parsing the "insecure" flag from the Data Prepper YAML. This translates to a "verify" flag that is passed to the Python requests library. This commit also includes unit tests for this logic. Signed-off-by: Kartik Ganesh --- index_configuration_tool/index_operations.py | 4 ++-- index_configuration_tool/main.py | 16 +++++++++++++++- index_configuration_tool/tests/test_main.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/index_configuration_tool/index_operations.py b/index_configuration_tool/index_operations.py index 51fa0e19a..22385c5ad 100644 --- a/index_configuration_tool/index_operations.py +++ b/index_configuration_tool/index_operations.py @@ -11,9 +11,9 @@ __INTERNAL_SETTINGS_KEYS = ["creation_date", "uuid", "provided_name", "version", "store"] -def fetch_all_indices(endpoint: str, optional_auth: Optional[tuple] = None) -> dict: +def fetch_all_indices(endpoint: str, optional_auth: Optional[tuple] = None, verify: bool = True) -> dict: actual_endpoint = endpoint + __ALL_INDICES_ENDPOINT - resp = requests.get(actual_endpoint, auth=optional_auth) + resp = requests.get(actual_endpoint, auth=optional_auth, verify=verify) # Remove internal settings result = dict(resp.json()) for index in result: diff --git a/index_configuration_tool/main.py b/index_configuration_tool/main.py index 944354068..87b84fe24 100644 --- a/index_configuration_tool/main.py +++ b/index_configuration_tool/main.py @@ -12,6 +12,18 @@ HOSTS_KEY = "hosts" USER_KEY = "username" PWD_KEY = "password" +INSECURE_KEY = "insecure" +CONNECTION_KEY = "connection" + + +# This config key may be either directly in the main dict (for sink) +# or inside a nested dict (for source). The default value is False. +def is_insecure(config: dict) -> bool: + if INSECURE_KEY in config: + return bool(config[INSECURE_KEY]) + elif CONNECTION_KEY in config and INSECURE_KEY in config[CONNECTION_KEY]: + return bool(config[CONNECTION_KEY][INSECURE_KEY]) + return False # TODO Only supports basic auth for now @@ -30,7 +42,9 @@ def get_endpoint_info(plugin_config: dict) -> tuple: def fetch_all_indices_by_plugin(plugin_config: dict) -> dict: endpoint, auth_tuple = get_endpoint_info(plugin_config) - return index_operations.fetch_all_indices(endpoint, auth_tuple) + # verify boolean will be the inverse of the insecure SSL key, if present + should_verify = not is_insecure(plugin_config) + return index_operations.fetch_all_indices(endpoint, auth_tuple, should_verify) def check_supported_endpoint(config: dict) -> Optional[tuple]: diff --git a/index_configuration_tool/tests/test_main.py b/index_configuration_tool/tests/test_main.py index cde419328..d2d0ea223 100644 --- a/index_configuration_tool/tests/test_main.py +++ b/index_configuration_tool/tests/test_main.py @@ -10,6 +10,8 @@ # Constants TEST_KEY = "test_key" +INSECURE_KEY = "insecure" +CONNECTION_KEY = "connection" BASE_CONFIG_SECTION = { TEST_KEY: [{"invalid_plugin1": {"key": "val"}}, {"invalid_plugin2": {}}] } @@ -43,6 +45,21 @@ def setUp(self) -> None: with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f: self.loaded_pipeline_config = pickle.load(f) + def test_is_insecure_default_value(self): + self.assertFalse(main.is_insecure({})) + + def test_is_insecure_top_level_key(self): + test_input = {"key": 123, INSECURE_KEY: True} + self.assertTrue(main.is_insecure(test_input)) + + def test_is_insecure_nested_key(self): + test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val", INSECURE_KEY: True}} + self.assertTrue(main.is_insecure(test_input)) + + def test_is_insecure_missing_nested(self): + test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}} + self.assertFalse(main.is_insecure(test_input)) + def test_get_auth_returns_none(self): # The following inputs should not return an auth tuple: # - Empty input From b392d4a09d4c6e75783522b4a7cd705e2c2cbdf3 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Tue, 27 Jun 2023 15:40:58 -0700 Subject: [PATCH 2/3] Better error messages in pipeline config validation Separated the error messages for source and sink validation. Also fixed the unit test for this. Signed-off-by: Kartik Ganesh --- index_configuration_tool/main.py | 6 ++++-- index_configuration_tool/tests/test_main.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/index_configuration_tool/main.py b/index_configuration_tool/main.py index 87b84fe24..781e13f66 100644 --- a/index_configuration_tool/main.py +++ b/index_configuration_tool/main.py @@ -84,8 +84,10 @@ def validate_plugin_config(config: dict, key: str): def validate_pipeline_config(config: dict): - if SOURCE_KEY not in config or SINK_KEY not in config: - raise ValueError("Missing source or sink configuration in Data Prepper pipeline YAML") + if SOURCE_KEY not in config: + raise ValueError("Missing source configuration in Data Prepper pipeline YAML") + if SINK_KEY not in config: + raise ValueError("Missing sink configuration in Data Prepper pipeline YAML") validate_plugin_config(config, SOURCE_KEY) validate_plugin_config(config, SINK_KEY) diff --git a/index_configuration_tool/tests/test_main.py b/index_configuration_tool/tests/test_main.py index d2d0ea223..3174abb96 100644 --- a/index_configuration_tool/tests/test_main.py +++ b/index_configuration_tool/tests/test_main.py @@ -203,7 +203,7 @@ def test_validate_pipeline_config_missing_required_keys(self): # - Empty input # - missing output # - missing input - bad_configs = [{}, {"input": ()}, {"output": ()}] + bad_configs = [{}, {"source": {}}, {"sink": {}}] for config in bad_configs: self.assertRaises(ValueError, main.validate_pipeline_config, config) From cc3f56e8522efd89f44a304eb704e2e731e97578 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Tue, 27 Jun 2023 15:51:44 -0700 Subject: [PATCH 3/3] Incorporated PR comments Fixed unnecessary bool() cast Signed-off-by: Kartik Ganesh --- index_configuration_tool/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index_configuration_tool/main.py b/index_configuration_tool/main.py index 781e13f66..8a1cdbf61 100644 --- a/index_configuration_tool/main.py +++ b/index_configuration_tool/main.py @@ -20,9 +20,9 @@ # or inside a nested dict (for source). The default value is False. def is_insecure(config: dict) -> bool: if INSECURE_KEY in config: - return bool(config[INSECURE_KEY]) + return config[INSECURE_KEY] elif CONNECTION_KEY in config and INSECURE_KEY in config[CONNECTION_KEY]: - return bool(config[CONNECTION_KEY][INSECURE_KEY]) + return config[CONNECTION_KEY][INSECURE_KEY] return False