From 06270925e2b96762863c1c760ac193fdc91e5b2b Mon Sep 17 00:00:00 2001 From: Florian Zipperle Date: Mon, 17 Jul 2023 21:26:27 +0200 Subject: [PATCH 1/3] Fix ndjson parser to store JSON fields correctly under target --- CHANGELOG-developer.next.asciidoc | 1 + libbeat/reader/parser/parser_test.go | 48 ++++++++++++++++++++++++++++ libbeat/reader/readjson/json.go | 4 ++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 751afbb13b0..23a6fd29341 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -83,6 +83,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fix the multiple host support for mongodb module {pull}34624[34624] - Skip HTTPJSON flakey test. {issue}34929[34929] {pull}35138[35138] - Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757] +- Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395] ==== Added diff --git a/libbeat/reader/parser/parser_test.go b/libbeat/reader/parser/parser_test.go index 50b416a11d4..d49cf3f2fe0 100644 --- a/libbeat/reader/parser/parser_test.go +++ b/libbeat/reader/parser/parser_test.go @@ -367,6 +367,54 @@ func TestJSONParsersWithFields(t *testing.T) { }, }, }, + "JSON post processor with dotted target key": { + message: reader.Message{ + Content: []byte("{\"key\":\"value\"}"), + Fields: mapstr.M{}, + }, + config: map[string]interface{}{ + "parsers": []map[string]interface{}{ + map[string]interface{}{ + "ndjson": map[string]interface{}{ + "target": "kubernetes.audit", + }, + }, + }, + }, + expectedMessage: reader.Message{ + Content: []byte(""), + Fields: mapstr.M{ + "kubernetes": mapstr.M{ + "audit": mapstr.M{ + "key": "value", + }, + }, + }, + }, + }, + "JSON post processor with non-dotted target key": { + message: reader.Message{ + Content: []byte("{\"key\":\"value\"}"), + Fields: mapstr.M{}, + }, + config: map[string]interface{}{ + "parsers": []map[string]interface{}{ + map[string]interface{}{ + "ndjson": map[string]interface{}{ + "target": "kubernetes", + }, + }, + }, + }, + expectedMessage: reader.Message{ + Content: []byte(""), + Fields: mapstr.M{ + "kubernetes": mapstr.M{ + "key": "value", + }, + }, + }, + }, "JSON post processor with document ID": { message: reader.Message{ Content: []byte("{\"key\":\"value\", \"my-id-field\":\"my-id\"}"), diff --git a/libbeat/reader/readjson/json.go b/libbeat/reader/readjson/json.go index f3690a79bb7..340503ec8f7 100644 --- a/libbeat/reader/readjson/json.go +++ b/libbeat/reader/readjson/json.go @@ -195,7 +195,9 @@ func (p *JSONParser) Next() (reader.Message, error) { message.Fields = event.Fields message.Meta = event.Meta } else { - message.AddFields(mapstr.M{p.target: jsonFields}) + fields := mapstr.M{} + fields.Put(p.target, jsonFields) + message.AddFields(fields) } return message, err From c14b86a1a2db336c326c16f3d73c9cddf516eaba Mon Sep 17 00:00:00 2001 From: Florian Zipperle Date: Wed, 19 Jul 2023 21:39:55 +0200 Subject: [PATCH 2/3] Update CHANGELOG.next.asciidoc --- CHANGELOG-developer.next.asciidoc | 1 - CHANGELOG.next.asciidoc | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 23a6fd29341..751afbb13b0 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -83,7 +83,6 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fix the multiple host support for mongodb module {pull}34624[34624] - Skip HTTPJSON flakey test. {issue}34929[34929] {pull}35138[35138] - Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757] -- Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395] ==== Added diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8b01cf6c662..bb9d8e711d6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -257,6 +257,8 @@ automatic splitting at root level, if root level element is an array. {pull}3415 *Elastic Logging Plugin* +*Libbeat* +- Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395] ==== Added From 61584976f29c2d1e9009b7acce5072a436c8950e Mon Sep 17 00:00:00 2001 From: Florian Zipperle Date: Thu, 20 Jul 2023 07:33:49 +0200 Subject: [PATCH 3/3] Move the change log entry to the right section --- CHANGELOG.next.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bb9d8e711d6..2253169371b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -74,6 +74,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix recovering from invalid output configuration when running under Elastic-Agent {pull}36016[36016] - Improve StreamBuf append to improve performance when reading long lines from files. {pull}35928[35928] - Eliminate cloning of event in deepUpdate {pull}35945[35945] +- Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395] *Auditbeat* @@ -257,8 +258,6 @@ automatic splitting at root level, if root level element is an array. {pull}3415 *Elastic Logging Plugin* -*Libbeat* -- Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395] ==== Added