diff --git a/CHANGELOG.md b/CHANGELOG.md index eab580502..e4d9f670e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Changed +- add regex and option to negate conditional profiles search + ## [1.9.2] ### Changed diff --git a/docs/configuration/configuring-profiles.md b/docs/configuration/configuring-profiles.md index cda3acdc0..26d345395 100644 --- a/docs/configuration/configuring-profiles.md +++ b/docs/configuration/configuring-profiles.md @@ -257,6 +257,31 @@ conditions: - 0 ``` +5. `regex` - value gathered from `field` match the pattern provided in `value`. +You can add options for regular expression after `/`. Possible options match ones used in [mongodb regex operator](https://www.mongodb.com/docs/manual/reference/operator/query/regex/). + +```yaml +conditions: + - field: IF-MIB.ifAdminStatus + operation: "regex" + value: ".own/i" +``` + +To negate operation you can add flag `negate_operation: "true"` to specified `field`. +```yaml +conditions: + - field: IF-MIB.ifAdminStatus + operation: "equals" + value: "up" + negate_operation: "true" +``` +It will negate the operator specified in `operation`. Possible negation: +1. `negate_operation + equals` - value gathered from `field` is NOT equal to `value` +2. `negate_operation + gt` - value gathered from `field` is SMALLER or EQUAL to `value` (works only for numeric values) +3. `negate_operation + lt` - value gathered from `field` is BIGGER or EQUAL to `value` (works only for numeric values) +4. `negate_operation + in` - value gathered from `field` is NOT equal to any of the elements provided in `value` +5. `negate_operation + regex` - value gathered from `field` is NOT matching the pattern provided in `value`. + `field` part of `conditions` must fulfill the pattern `MIB-family.field`. Fields must represent textual value (not metric one), you can learn more about it [here](snmp-data-format.md). diff --git a/integration_tests/test_poller_integration.py b/integration_tests/test_poller_integration.py index 35de382f6..f74492e1b 100644 --- a/integration_tests/test_poller_integration.py +++ b/integration_tests/test_poller_integration.py @@ -771,6 +771,336 @@ def test_equals_profile(self, request, setup_splunk): assert metric_count > 0 +@pytest.fixture(scope="class") +def setup_single_regex_and_options_profiles(request): + """ + Expected values for IF-MIB.ifDescr: + - IF-MIB.ifDescr = lo + - IF-MIB.ifDescr = eth0 + + regex_profile should result in polling IF-MIB.ifDescr + options_profile should result in polling IF-MIB.ifDescr + """ + trap_external_ip = request.config.getoption("trap_external_ip") + profiles = { + "regex_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifDescr", + "operation": dq("regex"), + "value": dq(".o"), + } + ], + }, + "options_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifDescr", + "operation": dq("regex"), + "value": dq(".TH0/i"), + } + ], + }, + } + + update_profiles(profiles) + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,regex_profile;options_profile,,", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml", "profiles.yaml"]) + time.sleep(120) + yield + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,regex_profile;options_profile,,t", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml"]) + time.sleep(120) + + +@pytest.mark.usefixtures("setup_single_regex_and_options_profiles") +class TestSingleRegexCorrectCondition: + def test_regex_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=regex_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + def test_regex_with_options_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=options_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + +@pytest.fixture(scope="class") +def setup_single_gt_and_lt_profiles_with_negation(request): + """ + Expected values for IF-MIB.ifIndex: + - IF-MIB.ifIndex.1 = 21 + - IF-MIB.ifIndex.2 = 10 + + not_gt_profile should result in polling IF-MIB.ifOutDiscards.1 + not_lt_profile should result in polling IF-MIB.ifOutDiscards.2 + """ + trap_external_ip = request.config.getoption("trap_external_ip") + profiles = { + "not_gt_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifIndex", + "operation": dq("gt"), + "value": 20, + "negate_operation": dq("true"), + } + ], + }, + "not_lt_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifIndex", + "operation": dq("lt"), + "value": 20, + "negate_operation": dq("true"), + } + ], + }, + } + + update_profiles(profiles) + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,not_gt_profile;not_lt_profile,,", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml", "profiles.yaml"]) + time.sleep(120) + yield + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,not_gt_profile;not_lt_profile,,t", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml"]) + time.sleep(120) + + +@pytest.mark.usefixtures("setup_single_gt_and_lt_profiles_with_negation") +class TestSingleGtAndLtWithNegationCorrectCondition: + def test_not_gt_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=not_gt_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + def test_not_lt_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=not_lt_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + +@pytest.fixture(scope="class") +def setup_single_in_and_equals_profiles_with_negation(request): + """ + Expected values for IF-MIB.ifDescr: + - IF-MIB.ifDescr.1 = lo + - IF-MIB.ifDescr.2 = eth0 + + not_in_profile should result in polling IF-MIB.ifOutDiscards.2 + not_equals_profile should result in polling IF-MIB.ifOutDiscards.1 + """ + trap_external_ip = request.config.getoption("trap_external_ip") + profiles = { + "not_in_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifDescr", + "operation": dq("in"), + "value": [dq("lo"), dq("test value")], + "negate_operation": dq("true"), + } + ], + }, + "not_equals_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifDescr", + "operation": dq("equals"), + "value": dq("eth0"), + "negate_operation": dq("true"), + } + ], + }, + } + + update_profiles(profiles) + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,not_in_profile;not_equals_profile,,", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml", "profiles.yaml"]) + time.sleep(120) + yield + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,not_in_profile;not_equals_profile,,t", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml"]) + time.sleep(120) + + +@pytest.mark.usefixtures("setup_single_in_and_equals_profiles_with_negation") +class TestSingleInAndEqualsWithNegationCorrectCondition: + def test_not_in_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=not_in_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + def test_not_equals_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=not_equals_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + +@pytest.fixture(scope="class") +def setup_single_regex_and_options_profiles_with_negation(request): + """ + Expected values for IF-MIB.ifDescr: + - IF-MIB.ifDescr = lo + - IF-MIB.ifDescr = eth0 + + not_regex_profile should result in polling IF-MIB.ifDescr + not_options_profile should result in polling IF-MIB.ifDescr + """ + trap_external_ip = request.config.getoption("trap_external_ip") + profiles = { + "not_regex_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifDescr", + "operation": dq("regex"), + "value": dq("e.h0"), + "negate_operation": dq("true"), + } + ], + }, + "not_options_profile": { + "frequency": 7, + "varBinds": [yaml_escape_list(sq("IF-MIB"), sq("ifOutDiscards"))], + "conditions": [ + { + "field": "IF-MIB.ifDescr", + "operation": dq("regex"), + "value": dq("L./i"), + "negate_operation": dq("true"), + } + ], + }, + } + + update_profiles(profiles) + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,not_regex_profile;not_options_profile,,", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml", "profiles.yaml"]) + time.sleep(120) + yield + update_file( + [ + f"{trap_external_ip},1166,2c,public,,,600,not_regex_profile;not_options_profile,,t", + ], + "inventory.yaml", + ) + upgrade_helm(["inventory.yaml"]) + time.sleep(120) + + +@pytest.mark.usefixtures("setup_single_regex_and_options_profiles_with_negation") +class TestSingleRegexWithNegationCorrectCondition: + def test_not_regex_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=not_regex_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + def test_not_regex_with_options_profile(self, request, setup_splunk): + time.sleep(20) + search_string = ( + """| mpreview index=netmetrics | search profiles=not_options_profile """ + ) + result_count, metric_count = run_retried_single_search( + setup_splunk, search_string, 2 + ) + assert result_count > 0 + assert metric_count > 0 + + @pytest.fixture(scope="class") def setup_multiple_conditions_profiles(request): """ diff --git a/splunk_connect_for_snmp/inventory/tasks.py b/splunk_connect_for_snmp/inventory/tasks.py index 02422d203..7d427d467 100644 --- a/splunk_connect_for_snmp/inventory/tasks.py +++ b/splunk_connect_for_snmp/inventory/tasks.py @@ -284,12 +284,20 @@ def create_profile(profile_name, frequency, varBinds, records): def create_query(conditions: typing.List[dict], address: str) -> dict: - conditional_profiles_mapping = { "equals": "$eq", "gt": "$gt", "lt": "$lt", "in": "$in", + "regex": "$regex", + } + + negative_profiles_mapping = { + "equals": "$ne", + "gt": "$lte", + "lt": "$gte", + "in": "$nin", + "regex": "$regex", } def _parse_mib_component(field: str) -> str: @@ -307,13 +315,33 @@ def _convert_to_float(value: typing.Any, ignore_error=False) -> typing.Any: else: raise BadlyFormattedFieldError(f"Value '{value}' should be numeric") + def _prepare_regex(value: str) -> typing.Union[list, str]: + pattern = value.strip("/").split("/") + if len(pattern) > 1: + return pattern + else: + return pattern[0] + def _get_value_for_operation(operation: str, value: str) -> typing.Any: if operation in ["lt", "gt"]: return _convert_to_float(value) elif operation == "in": return [_convert_to_float(v, True) for v in value] + elif operation == "regex": + return _prepare_regex(value) return value + def _prepare_query_input( + operation: str, value: typing.Any, field: str, negate_operation: bool + ) -> dict: + if operation == "regex" and type(value) == list: + query = {mongo_operation: value[0], "$options": value[1]} + else: + query = {mongo_operation: value} + if operation == "regex" and negate_operation: + query = {"$not": query} + return {f"fields.{field}.value": query} + filters = [] field = "" for condition in conditions: @@ -321,10 +349,20 @@ def _get_value_for_operation(operation: str, value: str) -> typing.Any: # fields in databases are written in convention "IF-MIB|ifInOctets" field = field.replace(".", "|") value = condition["value"] + negate_operation = human_bool( + condition.get("negate_operation", False), default=False + ) operation = condition["operation"].lower() value_for_querying = _get_value_for_operation(operation, value) - mongo_operation = conditional_profiles_mapping.get(operation) - filters.append({f"fields.{field}.value": {mongo_operation: value_for_querying}}) + mongo_operation = ( + negative_profiles_mapping.get(operation) + if negate_operation + else conditional_profiles_mapping.get(operation) + ) + query = _prepare_query_input( + operation, value_for_querying, field, negate_operation + ) + filters.append(query) mib_component = _parse_mib_component(field) return { "$and": [ diff --git a/test/inventory/test_conditional_profiles.py b/test/inventory/test_conditional_profiles.py index b62e49a3b..5754f6462 100644 --- a/test/inventory/test_conditional_profiles.py +++ b/test/inventory/test_conditional_profiles.py @@ -67,6 +67,168 @@ def test_single_in_condition(self, return_all_profiles): } self.assertEqual(create_query(conditions, address), expected_query) + def test_single_regex_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + {"field": "MIB-FAMILY.field1", "value": ".p", "operation": "regex"} + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + {"fields.MIB-FAMILY|field1.value": {"$regex": ".p"}}, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + + def test_single_regex_with_options_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + {"field": "MIB-FAMILY.field1", "value": "/.p/s", "operation": "regex"} + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + {"fields.MIB-FAMILY|field1.value": {"$regex": ".p", "$options": "s"}}, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + + def test_single_equals_negate_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + { + "field": "MIB-FAMILY.field1", + "value": "value1", + "operation": "equals", + "negate_operation": "true", + } + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + {"fields.MIB-FAMILY|field1.value": {"$ne": "value1"}}, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + + def test_single_lt_negate_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + { + "field": "MIB-FAMILY.field2", + "value": "10", + "operation": "lt", + "negate_operation": "true", + } + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + {"fields.MIB-FAMILY|field2.value": {"$gte": 10.0}}, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + + def test_single_gt_negate_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + { + "field": "MIB-FAMILY.field3", + "value": "20", + "operation": "gt", + "negate_operation": "true", + } + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + {"fields.MIB-FAMILY|field3.value": {"$lte": 20.0}}, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + + def test_single_in_negate_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + { + "field": "MIB-FAMILY.field4", + "value": [1, 2, 3], + "operation": "in", + "negate_operation": "true", + } + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + {"fields.MIB-FAMILY|field4.value": {"$nin": [1.0, 2.0, 3.0]}}, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + + def test_single_regex_negate_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + { + "field": "MIB-FAMILY.field1", + "value": ".own$", + "operation": "regex", + "negate_operation": "true", + } + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + {"fields.MIB-FAMILY|field1.value": {"$not": {"$regex": ".own$"}}}, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + + def test_single_regex_with_options_negate_condition(self, return_all_profiles): + from splunk_connect_for_snmp.inventory.tasks import create_query + + conditions = [ + { + "field": "MIB-FAMILY.field1", + "value": "/.own$/s", + "operation": "regex", + "negate_operation": "true", + } + ] + address = "127.0.0.1" + expected_query = { + "$and": [ + {"address": address}, + {"group_key_hash": {"$regex": "^MIB-FAMILY"}}, + { + "fields.MIB-FAMILY|field1.value": { + "$not": {"$regex": ".own$", "$options": "s"} + } + }, + ] + } + self.assertEqual(create_query(conditions, address), expected_query) + def test_multiple_conditions(self, return_all_profiles): from splunk_connect_for_snmp.inventory.tasks import create_query