From 2e65dedab2c811ef57a7375efd48bdc11f56b25b Mon Sep 17 00:00:00 2001 From: Victor Silva Date: Thu, 21 Sep 2023 18:10:33 -0300 Subject: [PATCH] feat: add transform to field details --- .gitignore | 3 + .../test_dynatrace_3th_file_configuration.py | 81 +++++++++++++++++++ .../fluentd/etc/fluent-dynatrace-3.conf | 81 +++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 tests/integration/fluentd/test_dynatrace_3th_file_configuration.py create mode 100644 tests/resources/fluentd-setup/fluentd/etc/fluent-dynatrace-3.conf diff --git a/.gitignore b/.gitignore index adc270a..65a9252 100644 --- a/.gitignore +++ b/.gitignore @@ -251,3 +251,6 @@ build-iPhoneSimulator/ # .rubocop-https?--* # End of https://www.gitignore.io/api/ruby,python,intellij+all + +# Vim +*.swp diff --git a/tests/integration/fluentd/test_dynatrace_3th_file_configuration.py b/tests/integration/fluentd/test_dynatrace_3th_file_configuration.py new file mode 100644 index 0000000..db4712d --- /dev/null +++ b/tests/integration/fluentd/test_dynatrace_3th_file_configuration.py @@ -0,0 +1,81 @@ +from datetime import datetime +from uuid import uuid4 + +import pytest +from faker import Faker + +from python_fluentd_testing.fluentd_evaluator import FluentdEvaluator +from tests.resources.support import absolute_path_fluentd_output_file +from tests.resources.support import try_to_remove_key_otherwise_return_it + + +@pytest.fixture +def setup_fluentd_scenario_3(): + folder_location, abs_file_path = absolute_path_fluentd_output_file("fluent-dynatrace-output-3.log") + with FluentdEvaluator( + "fluent-dynatrace-3.conf", folder_location, abs_file_path, 24232 + ).initialize_fluent_daemon() as f: + yield f + + +def test_should_emit_and_transform_content_and_body_keys_to_send_to_dynatrace(setup_fluentd_scenario_3): + # Arrange + sample_stream_payload = [ + { + "log_id": "90020221124231024915274964892613219266671587159978278962", + "data": { + "date": "2022-11-24T23:10:23.560Z", + "type": "fp", + "description": "Wrong email or password.", + "connection": "jsm-main-including-migrations", + "connection_id": "con_DtzeqpeWHFrePk99", + "client_id": "DMx4w5HbYCCWzAXRxZmK2pPJMyXzYPjx", + "client_name": "Loja Virtual", + "ip": "2804:14d:1a87:cf49::9311", + "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "details": { + "error": { + "message": "Wrong email or password." + }, + "request": { + "method": "requestOptions", + "path": "requestOptions", + }, + "response": { + "body": "httpResponseBody", + "statusCode": "httpResponseStatusCode", + }, + }, + "user_id": "auth0|45f28abf-e4d6-4001-a29a-0a00a38e87b2", + "user_name": "cleverson.purkot@gmail.com", + "strategy": "auth0", + "strategy_type": "database", + "log_id": "90020221124231024915274964892613219266671587159978278962", + }, + } + ] + # Act + result = setup_fluentd_scenario_3.emit_it_through_http_post_and_get_computed_result(sample_stream_payload) + # Assert + cleaned_result = try_to_remove_key_otherwise_return_it(result, "tag") + # https://www.dynatrace.com/support/help/dynatrace-api/environment-api/log-monitoring-v2/post-ingest-logs + # Content. If the content key is not set, the whole JSON is parsed as the content. + auth0_event = sample_stream_payload[0]["data"] + expected_namespace = f'{auth0_event["strategy"]}|{auth0_event["strategy_type"]}|{auth0_event["connection"]}' + assert cleaned_result == { + "content": "Wrong email or password.", + "body": { + "error": {"message": "Wrong email or password."}, + "request": {"method": "requestOptions", "path": "requestOptions"}, + "response": {"body": "httpResponseBody", "statusCode": "httpResponseStatusCode"} + }, + "date": auth0_event["date"], + "severity": "alert", + "service.name": "auth0", + "service.namespace": expected_namespace, + "audit.action": auth0_event["type"], + "audit.identity": auth0_event["user_id"], + "audit.result": auth0_event["details"]["error"]["message"], + "trace_id": sample_stream_payload[0]["log_id"], + "log.source": "fluentd", + } diff --git a/tests/resources/fluentd-setup/fluentd/etc/fluent-dynatrace-3.conf b/tests/resources/fluentd-setup/fluentd/etc/fluent-dynatrace-3.conf new file mode 100644 index 0000000..866526f --- /dev/null +++ b/tests/resources/fluentd-setup/fluentd/etc/fluent-dynatrace-3.conf @@ -0,0 +1,81 @@ + + @type http + port 24232 + bind 0.0.0.0 + + + + type split_array + + + + @type rewrite_tag_filter + + key $.data.type + pattern /^du|f|fapi|fc|fce|fco|fcoa|fcp|fcph|fcpn|fcpr|fcpro|fcu|fd|fdeac|fdeaz|fdecc|fdu|feacft|feccft|fede|fens|feoobft|feotpft|fepft|fepotpft|fercft|fertft|ferrt|fi|flo|fn|fp|fs|fsa|fu|fui|fv|fvr|gd_auth_failed|gd_otp_rate_limit_exceed|gd_recovery_rate_limit_exceed|gd_send_sms_failure|gd_send_voice_failure|gd_start_enroll_failed|gd_webauthn_enrollment_failed$/ + tag alert.label.jsm + + + # Catch all rule so it does not drop the record + key $.data.type + pattern .* + tag unknown.label.jsm + + + + + @type record_modifier + + severity 'alert' + + + + + @type record_modifier + remove_keys _dummy_ + + _dummy_ ${if record['data'].has_key?('description'); record['content'] = record['data']['description']; elsif record['data'].has_key?('details') and record['data']['details'].has_key?('error') and record['data']['details']['error'].has_key?('message'); record['content'] = record['data']['details']['error']['message']; else record['content'] = 'N/A'; end; nil} + + + + + @type record_modifier + remove_keys _dummy_ + + _dummy_ ${if record['data'].has_key?('details'); record['detail'] = record['data']['details']; else record['detail'] = 'N/A'; end; nil} + + + + + @type record_transformer + enable_ruby + renew_record + + body ${record["detail"]} + content ${record["content"]} + date ${record["data"]["date"]} + severity ${record["severity"]} + service.name 'auth0' + service.namespace ${record["data"]["strategy"]}|${record["data"]["strategy_type"]}|${record["data"]["connection"]} + audit.action ${record["data"]["type"]} + audit.identity ${record["data"]["user_id"]} + audit.result ${record["data"]["details"]["error"]["message"]} + trace_id ${record["log_id"]} + log.source 'fluentd' + + + + + @type file + path /result/fluent-dynatrace-output-3.log + append true + add_path_suffix false + + @type json + + + @type memory + chunk_limit_records 1 + retry_max_times 0 + +