From 32c7305f81782a38415f6e17847757e04e89298b Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 28 Nov 2024 21:58:35 +0000 Subject: [PATCH 1/6] [exporter/elasticsearch] Merge geo.location.{lat,lon} to geo.location --- exporter/elasticsearchexporter/model.go | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index 299cb3902347..deb29f180dad 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -998,3 +998,29 @@ func sliceHash(h hash.Hash, s pcommon.Slice) { valueHash(h, s.At(i)) } } + +// convertGeolocationToGeopoint mutates attributes map to merge `geo.location.lat` and `geo.location.lon` to `geo.location`. +func convertGeolocationToGeopoint(attributes pcommon.Map) { + const ( + lonKey = "geo.location.lon" + latKey = "geo.location.lat" + mergedKey = "geo.location" + ) + var lon, lat pcommon.Value + if v, ok := attributes.Get(lonKey); ok { + lon = v + } + if v, ok := attributes.Get(latKey); ok { + lat = v + } + if lon.Type() == pcommon.ValueTypeDouble && lat.Type() == pcommon.ValueTypeDouble { + attributes.PutStr(mergedKey, fmt.Sprintf("POINT(%f %f)", lon.Double(), lat.Double())) + attributes.RemoveIf(func(key string, val pcommon.Value) bool { + switch key { + case lonKey, latKey: + return true + } + return false + }) + } +} From 211450ed60e3381b44a5a2d1602ff2ba9a41f2d4 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 29 Nov 2024 11:33:36 +0000 Subject: [PATCH 2/6] Handle namespace --- exporter/elasticsearchexporter/model.go | 80 ++++++++++++++++---- exporter/elasticsearchexporter/model_test.go | 33 ++++++++ 2 files changed, 98 insertions(+), 15 deletions(-) diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index deb29f180dad..8458d13f42b8 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -13,6 +13,7 @@ import ( "hash/fnv" "math" "slices" + "strings" "time" jsoniter "github.com/json-iterator/go" @@ -999,28 +1000,77 @@ func sliceHash(h hash.Hash, s pcommon.Slice) { } } -// convertGeolocationToGeopoint mutates attributes map to merge `geo.location.lat` and `geo.location.lon` to `geo.location`. -func convertGeolocationToGeopoint(attributes pcommon.Map) { +// mergeGeolocation mutates attributes map to merge all `geo.location.{lon,lat}`, +// and namespaced `*.geo.location.{lon,lat}` to unnamespaced and namespaced `geo.location`. +// This is to match the geo_point type in Elasticsearch. +func mergeGeolocation(attributes pcommon.Map) { const ( lonKey = "geo.location.lon" latKey = "geo.location.lat" mergedKey = "geo.location" ) - var lon, lat pcommon.Value - if v, ok := attributes.Get(lonKey); ok { - lon = v + // Prefix is the attribute name without lonKey or latKey suffix + // e.g. prefix of "foo.bar.geo.location.lon" is "foo.bar.", prefix of "geo.location.lon" is "". + prefixToGeo := make(map[string]struct { + lon, lat float64 + lonSet, latSet bool + }) + setLon := func(prefix string, v float64) { + g := prefixToGeo[prefix] + g.lon = v + g.lonSet = true + prefixToGeo[prefix] = g } - if v, ok := attributes.Get(latKey); ok { - lat = v + setLat := func(prefix string, v float64) { + g := prefixToGeo[prefix] + g.lat = v + g.latSet = true + prefixToGeo[prefix] = g } - if lon.Type() == pcommon.ValueTypeDouble && lat.Type() == pcommon.ValueTypeDouble { - attributes.PutStr(mergedKey, fmt.Sprintf("POINT(%f %f)", lon.Double(), lat.Double())) - attributes.RemoveIf(func(key string, val pcommon.Value) bool { - switch key { - case lonKey, latKey: - return true - } + attributes.RemoveIf(func(key string, val pcommon.Value) bool { + if val.Type() != pcommon.ValueTypeDouble { return false - }) + } + v := val.Double() + + if key == lonKey { + setLon("", v) + return true + } else if key == latKey { + setLat("", v) + return true + } else if namespace, found := strings.CutSuffix(key, "."+lonKey); found { + prefix := namespace + "." + setLon(prefix, v) + return true + } else if namespace, found := strings.CutSuffix(key, "."+latKey); found { + prefix := namespace + "." + setLat(prefix, v) + return true + } + + return false + }) + + for prefix, geo := range prefixToGeo { + if geo.lonSet && geo.latSet { + key := prefix + mergedKey + // Geopoint expressed as an array with the format: [lon, lat] + s := attributes.PutEmptySlice(key) + s.EnsureCapacity(2) + s.AppendEmpty().SetDouble(geo.lon) + s.AppendEmpty().SetDouble(geo.lat) + continue + } + + // Place the attributes back if lon and lat are not present together + if geo.lonSet { + key := prefix + lonKey + attributes.PutDouble(key, geo.lon) + } + if geo.latSet { + key := prefix + latKey + attributes.PutDouble(key, geo.lat) + } } } diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index ad414ed275fe..9b28e2459068 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -1278,3 +1278,36 @@ func TestEncodeLogBodyMapMode(t *testing.T) { require.Error(t, err) require.ErrorIs(t, err, ErrInvalidTypeForBodyMapMode) } + +func TestMergeGeolocation(t *testing.T) { + attributes := map[string]any{ + "geo.location.lon": 1.1, + "geo.location.lat": 2.2, + "foo.bar.geo.location.lon": 3.3, + "foo.bar.geo.location.lat": 4.4, + "a.geo.location.lon": 5.5, + "b.geo.location.lat": 6.6, + "unrelatedgeo.location.lon": 7.7, + "unrelatedgeo.location.lat": 8.8, + "d": 9.9, + "e.geo.location.lon": "foo", + "e.geo.location.lat": "bar", + } + wantAttributes := map[string]any{ + "geo.location": []any{1.1, 2.2}, + "foo.bar.geo.location": []any{3.3, 4.4}, + "a.geo.location.lon": 5.5, + "b.geo.location.lat": 6.6, + "unrelatedgeo.location.lon": 7.7, + "unrelatedgeo.location.lat": 8.8, + "d": 9.9, + "e.geo.location.lon": "foo", + "e.geo.location.lat": "bar", + } + input := pcommon.NewMap() + err := input.FromRaw(attributes) + require.NoError(t, err) + mergeGeolocation(input) + after := input.AsRaw() + assert.Equal(t, wantAttributes, after) +} From 006f7c6623d45580eac2c590bab061d240f80a49 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 29 Nov 2024 11:35:15 +0000 Subject: [PATCH 3/6] Use mergeGeolocation in otel mode --- exporter/elasticsearchexporter/model.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index 8458d13f42b8..a17fd84241a5 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -600,7 +600,7 @@ func (m *encodeModel) encodeResourceOTelMode(document *objmodel.Document, resour } return false }) - + mergeGeolocation(resourceAttrMap) document.Add("resource", objmodel.ValueFromAttribute(resourceMapVal)) } @@ -626,6 +626,7 @@ func (m *encodeModel) encodeScopeOTelMode(document *objmodel.Document, scope pco } return false }) + mergeGeolocation(scopeAttrMap) document.Add("scope", objmodel.ValueFromAttribute(scopeMapVal)) } @@ -645,6 +646,7 @@ func (m *encodeModel) encodeAttributesOTelMode(document *objmodel.Document, attr } return false }) + mergeGeolocation(attrsCopy) document.AddAttributes("attributes", attrsCopy) } From 9a579393b5a63a9bf8de82f5cb4ec42af7dcbded Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 29 Nov 2024 11:40:54 +0000 Subject: [PATCH 4/6] Add changelog --- ...sticsearchexporter_merge-geo-location.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/elasticsearchexporter_merge-geo-location.yaml diff --git a/.chloggen/elasticsearchexporter_merge-geo-location.yaml b/.chloggen/elasticsearchexporter_merge-geo-location.yaml new file mode 100644 index 000000000000..cd6e5cdb09b2 --- /dev/null +++ b/.chloggen/elasticsearchexporter_merge-geo-location.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Merge *.geo.location.{lat,lon} to *.geo.location in OTel mode + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36565] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: In OTel mapping mode, merge *.geo.location.{lat,lon} to *.geo.location such that they are stored as geo_point in Elasticsearch. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] From 5603ab9654bddd74c6d3ffd84a0e5a517005824b Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Tue, 3 Dec 2024 14:34:35 +0000 Subject: [PATCH 5/6] Avoid val.Double call if key does not match --- exporter/elasticsearchexporter/model.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index a17fd84241a5..ba14a24d4fc6 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -1033,24 +1033,22 @@ func mergeGeolocation(attributes pcommon.Map) { if val.Type() != pcommon.ValueTypeDouble { return false } - v := val.Double() if key == lonKey { - setLon("", v) + setLon("", val.Double()) return true } else if key == latKey { - setLat("", v) + setLat("", val.Double()) return true } else if namespace, found := strings.CutSuffix(key, "."+lonKey); found { prefix := namespace + "." - setLon(prefix, v) + setLon(prefix, val.Double()) return true } else if namespace, found := strings.CutSuffix(key, "."+latKey); found { prefix := namespace + "." - setLat(prefix, v) + setLat(prefix, val.Double()) return true } - return false }) From 64982652a2c1549095a8f010bc1d42b1d49ee88e Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 4 Dec 2024 12:08:30 +0000 Subject: [PATCH 6/6] Update .chloggen/elasticsearchexporter_merge-geo-location.yaml Co-authored-by: Vishal Raj --- .chloggen/elasticsearchexporter_merge-geo-location.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/elasticsearchexporter_merge-geo-location.yaml b/.chloggen/elasticsearchexporter_merge-geo-location.yaml index cd6e5cdb09b2..23979918eb7a 100644 --- a/.chloggen/elasticsearchexporter_merge-geo-location.yaml +++ b/.chloggen/elasticsearchexporter_merge-geo-location.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: elasticsearchexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Merge *.geo.location.{lat,lon} to *.geo.location in OTel mode +note: Map *.geo.location.{lat,lon} as geo_point field in OTel mode # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [36565]