[Index configuration tool] Add support for insecure HTTPS endpoint

index_configuration_tool/index_operations.py

@@ -11,9 +11,9 @@
 __INTERNAL_SETTINGS_KEYS = ["creation_date", "uuid", "provided_name", "version", "store"]
 
 
-def fetch_all_indices(endpoint: str, optional_auth: Optional[tuple] = None) -> dict:
+def fetch_all_indices(endpoint: str, optional_auth: Optional[tuple] = None, verify: bool = True) -> dict:
     actual_endpoint = endpoint + __ALL_INDICES_ENDPOINT
-    resp = requests.get(actual_endpoint, auth=optional_auth)
+    resp = requests.get(actual_endpoint, auth=optional_auth, verify=verify)
     # Remove internal settings
     result = dict(resp.json())
     for index in result:

index_configuration_tool/main.py

@@ -12,6 +12,18 @@
 HOSTS_KEY = "hosts"
 USER_KEY = "username"
 PWD_KEY = "password"
+INSECURE_KEY = "insecure"
+CONNECTION_KEY = "connection"
+
+
+# This config key may be either directly in the main dict (for sink)
+# or inside a nested dict (for source). The default value is False.
+def is_insecure(config: dict) -> bool:
+    if INSECURE_KEY in config:
+        return config[INSECURE_KEY]
+    elif CONNECTION_KEY in config and INSECURE_KEY in config[CONNECTION_KEY]:
+        return config[CONNECTION_KEY][INSECURE_KEY]
+    return False
 
 
 # TODO Only supports basic auth for now
@@ -30,7 +42,9 @@ def get_endpoint_info(plugin_config: dict) -> tuple:
 
 def fetch_all_indices_by_plugin(plugin_config: dict) -> dict:
     endpoint, auth_tuple = get_endpoint_info(plugin_config)
-    return index_operations.fetch_all_indices(endpoint, auth_tuple)
+    # verify boolean will be the inverse of the insecure SSL key, if present
+    should_verify = not is_insecure(plugin_config)
+    return index_operations.fetch_all_indices(endpoint, auth_tuple, should_verify)
 
 
 def check_supported_endpoint(config: dict) -> Optional[tuple]:
@@ -70,8 +84,10 @@ def validate_plugin_config(config: dict, key: str):
 
 
 def validate_pipeline_config(config: dict):
-    if SOURCE_KEY not in config or SINK_KEY not in config:
-        raise ValueError("Missing source or sink configuration in Data Prepper pipeline YAML")
+    if SOURCE_KEY not in config:
+        raise ValueError("Missing source configuration in Data Prepper pipeline YAML")
+    if SINK_KEY not in config:
+        raise ValueError("Missing sink configuration in Data Prepper pipeline YAML")
     validate_plugin_config(config, SOURCE_KEY)
     validate_plugin_config(config, SINK_KEY)
 

index_configuration_tool/tests/test_main.py

@@ -10,6 +10,8 @@
 
 # Constants
 TEST_KEY = "test_key"
+INSECURE_KEY = "insecure"
+CONNECTION_KEY = "connection"
 BASE_CONFIG_SECTION = {
     TEST_KEY: [{"invalid_plugin1": {"key": "val"}}, {"invalid_plugin2": {}}]
 }
@@ -43,6 +45,21 @@ def setUp(self) -> None:
         with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f:
             self.loaded_pipeline_config = pickle.load(f)
 
+    def test_is_insecure_default_value(self):
+        self.assertFalse(main.is_insecure({}))
+
+    def test_is_insecure_top_level_key(self):
+        test_input = {"key": 123, INSECURE_KEY: True}
+        self.assertTrue(main.is_insecure(test_input))
+
+    def test_is_insecure_nested_key(self):
+        test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val", INSECURE_KEY: True}}
+        self.assertTrue(main.is_insecure(test_input))
+
+    def test_is_insecure_missing_nested(self):
+        test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}}
+        self.assertFalse(main.is_insecure(test_input))
+
     def test_get_auth_returns_none(self):
         # The following inputs should not return an auth tuple:
         # - Empty input
@@ -186,7 +203,7 @@ def test_validate_pipeline_config_missing_required_keys(self):
         # - Empty input
         # - missing output
         # - missing input
-        bad_configs = [{}, {"input": ()}, {"output": ()}]
+        bad_configs = [{}, {"source": {}}, {"sink": {}}]
         for config in bad_configs:
             self.assertRaises(ValueError, main.validate_pipeline_config, config)