From f9762e07cbb03870449f9bf8bc03bf188a217a62 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 14 Aug 2023 17:44:59 +0800 Subject: [PATCH] Update apm-data and apm-aggregation (#11386) --- NOTICE.txt | 32 +++++++------- changelogs/head.asciidoc | 3 ++ go.mod | 20 ++++----- go.sum | 42 +++++++++---------- internal/agentcfg/reporter.go | 4 +- internal/agentcfg/reporter_test.go | 2 +- internal/beater/api/mux.go | 7 ++-- internal/beater/api/mux_test.go | 8 ++-- internal/beater/processors.go | 4 +- internal/publish/pub.go | 2 +- internal/publish/pub_test.go | 3 +- .../TestCompressedSpans.approved.json | 3 -- .../approvals/TestDropUnsampled.approved.json | 3 -- testdata/intake-v2/spans.ndjson | 6 +-- testdata/intake-v2/transactions.ndjson | 2 +- x-pack/apm-server/aggregation/lsm.go | 1 - x-pack/apm-server/sampling/groups.go | 2 +- x-pack/apm-server/sampling/groups_test.go | 5 +-- x-pack/apm-server/sampling/processor_test.go | 23 +++++----- x-pack/apm-server/sampling/pubsub/pubsub.go | 3 +- 20 files changed, 82 insertions(+), 93 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 2ca5f0bddba..23a286233bd 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -260,11 +260,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-aggregation -Version: v0.0.0-20230807142825-c82b2b7e590c +Version: v0.0.0-20230814031111-fa6b81b35e94 Licence type (autodetected): Elastic-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-aggregation@v0.0.0-20230807142825-c82b2b7e590c/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-aggregation@v0.0.0-20230814031111-fa6b81b35e94/LICENSE.txt: Elastic License 2.0 @@ -362,11 +362,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-data -Version: v0.1.1-0.20230803060036-9180b59d7888 +Version: v0.1.1-0.20230814023104-48e4b20579f6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20230803060036-9180b59d7888/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20230814023104-48e4b20579f6/LICENSE: Apache License Version 2.0, January 2004 @@ -6369,11 +6369,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : go.uber.org/zap -Version: v1.24.0 +Version: v1.25.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.24.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.25.0/LICENSE.txt: Copyright (c) 2016-2017 Uber Technologies, Inc. @@ -6398,11 +6398,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.12.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -6472,11 +6472,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.10.0 +Version: v0.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.10.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.11.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -18031,11 +18031,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.11.0 +Version: v0.12.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.11.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.12.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -18105,11 +18105,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.10.0 +Version: v0.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.10.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.11.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -18142,11 +18142,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.11.0 +Version: v0.12.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.11.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.12.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 1c8eeba33ab..9b1e1ceaedd 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -5,6 +5,8 @@ https://github.com/elastic/apm-server/compare/8.9\...main[View commits] [float] ==== Breaking Changes +- Aggregated metrics now consider global labels to be part of a service's identity, and high cardinality global labels may cause services to be obscured {pull}11386[11386] +- Event protobuf encoding for tail-based sampling changed (to a more efficient encoding) for event timestamp and duration {pull}11386[11386] [float] ==== Deprecations @@ -14,6 +16,7 @@ https://github.com/elastic/apm-server/compare/8.9\...main[View commits] [float] ==== Intake API Changes +- `null` is no longer an acceptable value for `context.http.request.headers` and `context.http.response.headers` {pull}11386[11386] [float] ==== Added diff --git a/go.mod b/go.mod index fbdbd6996dc..9887a7d84a5 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b github.com/dustin/go-humanize v1.0.1 - github.com/elastic/apm-aggregation v0.0.0-20230807142825-c82b2b7e590c - github.com/elastic/apm-data v0.1.1-0.20230803060036-9180b59d7888 + github.com/elastic/apm-aggregation v0.0.0-20230814031111-fa6b81b35e94 + github.com/elastic/apm-data v0.1.1-0.20230814023104-48e4b20579f6 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230811045459-4506e55d50f9 github.com/elastic/elastic-agent-client/v7 v7.2.0 github.com/elastic/elastic-agent-libs v0.3.11 @@ -47,10 +47,10 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/sdk/metric v0.39.0 go.uber.org/automaxprocs v1.5.2 - go.uber.org/zap v1.24.0 - golang.org/x/net v0.12.0 + go.uber.org/zap v1.25.0 + golang.org/x/net v0.14.0 golang.org/x/sync v0.3.0 - golang.org/x/term v0.10.0 + golang.org/x/term v0.11.0 golang.org/x/time v0.3.0 google.golang.org/grpc v1.56.1 google.golang.org/protobuf v1.31.0 @@ -155,12 +155,12 @@ require ( go.opentelemetry.io/otel/trace v1.16.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.11.0 // indirect + golang.org/x/crypto v0.12.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect - golang.org/x/mod v0.11.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect - golang.org/x/tools v0.9.3 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/tools v0.12.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect diff --git a/go.sum b/go.sum index d6a18bb4b92..a71e656f590 100644 --- a/go.sum +++ b/go.sum @@ -150,10 +150,10 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/elastic/apm-aggregation v0.0.0-20230807142825-c82b2b7e590c h1:VKmJ07OWzS+mKx1ahY3XNUz/0iaixjQZGds64qrlUGY= -github.com/elastic/apm-aggregation v0.0.0-20230807142825-c82b2b7e590c/go.mod h1:TwDJCDT04Ni8WrSrD4oEBATnDa18dteY//jhL4kqGio= -github.com/elastic/apm-data v0.1.1-0.20230803060036-9180b59d7888 h1:R2HpM1KxROvzvMHq/3XmuT/Y6oTcTteUFMJtQCP04GM= -github.com/elastic/apm-data v0.1.1-0.20230803060036-9180b59d7888/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= +github.com/elastic/apm-aggregation v0.0.0-20230814031111-fa6b81b35e94 h1:UGK3WBPEFepBn5CRJVf5HZsN+OkUchuWaA8cNtl7u5Q= +github.com/elastic/apm-aggregation v0.0.0-20230814031111-fa6b81b35e94/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo= +github.com/elastic/apm-data v0.1.1-0.20230814023104-48e4b20579f6 h1:ixw/vSYEKysrW8xl8gcKEo443dySneWvKjtzsOq893s= +github.com/elastic/apm-data v0.1.1-0.20230814023104-48e4b20579f6/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230811045459-4506e55d50f9 h1:B8KxebUXsS4t1cqBdgx3H3LDKQA+YCflxc8o0EVayBc= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230811045459-4506e55d50f9/go.mod h1:CV/gSnBLMU4ie75Qq8C2JmOPdlvGyqQHg8GgrFxhqYM= github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+PiWyZ4I4F4LqrCYgYlNc= @@ -647,13 +647,13 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -665,8 +665,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -702,8 +702,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -744,8 +744,8 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -821,12 +821,12 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -834,8 +834,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -893,8 +893,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/agentcfg/reporter.go b/internal/agentcfg/reporter.go index 26e7930a60d..9b3943455cb 100644 --- a/internal/agentcfg/reporter.go +++ b/internal/agentcfg/reporter.go @@ -22,8 +22,6 @@ import ( "sync" "time" - "google.golang.org/protobuf/types/known/timestamppb" - "github.com/elastic/apm-data/model/modelpb" "github.com/elastic/elastic-agent-libs/logp" ) @@ -87,7 +85,7 @@ func (r Reporter) Run(ctx context.Context) error { batch := make(modelpb.Batch, 0, len(applied)) for etag := range applied { batch = append(batch, &modelpb.APMEvent{ - Timestamp: timestamppb.Now(), + Timestamp: modelpb.FromTime(time.Now()), Labels: modelpb.Labels{"etag": {Value: etag}}, Metricset: &modelpb.Metricset{ Name: "agent_config", diff --git a/internal/agentcfg/reporter_test.go b/internal/agentcfg/reporter_test.go index 66d73dc60fd..c912316f77f 100644 --- a/internal/agentcfg/reporter_test.go +++ b/internal/agentcfg/reporter_test.go @@ -74,7 +74,7 @@ func TestReportFetch(t *testing.T) { // Assert the timestamp is not empty and set the timestamp to an empty // value so we can assert equality in the list contents. assert.NotZero(t, received.Timestamp, "empty timestamp") - bp.received[i].Timestamp = nil + bp.received[i].Timestamp = 0 } // We use assert.ElementsMatch because the etags may not be diff --git a/internal/beater/api/mux.go b/internal/beater/api/mux.go index f870abc9f54..d86e407b1a2 100644 --- a/internal/beater/api/mux.go +++ b/internal/beater/api/mux.go @@ -26,7 +26,6 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "go.uber.org/zap" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/monitoring" @@ -297,7 +296,7 @@ func rootMiddleware(cfg *config.Config, authenticator *auth.Authenticator) []mid func baseRequestMetadata(c *request.Context) *modelpb.APMEvent { return &modelpb.APMEvent{ - Timestamp: timestamppb.New(c.Timestamp), + Timestamp: modelpb.FromTime(c.Timestamp), } } @@ -307,7 +306,7 @@ func backendRequestMetadataFunc(cfg *config.Config) func(c *request.Context) *mo } return func(c *request.Context) *modelpb.APMEvent { e := modelpb.APMEvent{ - Timestamp: timestamppb.New(c.Timestamp), + Timestamp: modelpb.FromTime(c.Timestamp), } if c.ClientIP.IsValid() { @@ -325,7 +324,7 @@ func rumRequestMetadataFunc(cfg *config.Config) func(c *request.Context) *modelp } return func(c *request.Context) *modelpb.APMEvent { e := modelpb.APMEvent{ - Timestamp: timestamppb.New(c.Timestamp), + Timestamp: modelpb.FromTime(c.Timestamp), } if c.UserAgent != "" { e.UserAgent = &modelpb.UserAgent{Original: c.UserAgent} diff --git a/internal/beater/api/mux_test.go b/internal/beater/api/mux_test.go index 792fc3eb876..ce0ca1d8f90 100644 --- a/internal/beater/api/mux_test.go +++ b/internal/beater/api/mux_test.go @@ -47,12 +47,12 @@ func TestBackendRequestMetadata(t *testing.T) { c := &request.Context{Timestamp: tNow} cfg := &config.Config{AugmentEnabled: true} event := backendRequestMetadataFunc(cfg)(c) - assert.Equal(t, tNow, event.Timestamp.AsTime()) + assert.Equal(t, tNow, modelpb.ToTime(event.Timestamp)) assert.Nil(t, nil, event.Host) c.ClientIP = netip.MustParseAddr("127.0.0.1") event = backendRequestMetadataFunc(cfg)(c) - assert.Equal(t, tNow, event.Timestamp.AsTime()) + assert.Equal(t, tNow, modelpb.ToTime(event.Timestamp)) assert.Equal(t, &modelpb.Host{Ip: []*modelpb.IP{modelpb.Addr2IP(c.ClientIP)}}, event.Host) } @@ -61,7 +61,7 @@ func TestRUMRequestMetadata(t *testing.T) { c := &request.Context{Timestamp: tNow} cfg := &config.Config{AugmentEnabled: true} event := rumRequestMetadataFunc(cfg)(c) - assert.Equal(t, tNow, event.Timestamp.AsTime()) + assert.Equal(t, tNow, modelpb.ToTime(event.Timestamp)) assert.Nil(t, event.Client) assert.Nil(t, event.Source) assert.Nil(t, event.UserAgent) @@ -69,7 +69,7 @@ func TestRUMRequestMetadata(t *testing.T) { ip := netip.MustParseAddr("127.0.0.1") c = &request.Context{Timestamp: tNow, ClientIP: ip, SourceIP: ip, UserAgent: "firefox"} event = rumRequestMetadataFunc(cfg)(c) - assert.Equal(t, tNow, event.Timestamp.AsTime()) + assert.Equal(t, tNow, modelpb.ToTime(event.Timestamp)) assert.Equal(t, &modelpb.Client{Ip: modelpb.Addr2IP(c.ClientIP)}, event.Client) assert.Equal(t, &modelpb.Source{Ip: modelpb.Addr2IP(c.SourceIP)}, event.Source) assert.Equal(t, &modelpb.UserAgent{Original: c.UserAgent}, event.UserAgent) diff --git a/internal/beater/processors.go b/internal/beater/processors.go index 7ab7fc98b85..74d57022085 100644 --- a/internal/beater/processors.go +++ b/internal/beater/processors.go @@ -89,8 +89,8 @@ func newObserverBatchProcessor() modelpb.ProcessBatchFunc { // data stream mappings, in 8.10. func removeEventReceivedBatchProcessor(ctx context.Context, batch *modelpb.Batch) error { for _, event := range *batch { - if event.Event != nil && event.Event.Received != nil { - event.Event.Received = nil + if event.Event != nil { + event.Event.Received = 0 } } return nil diff --git a/internal/publish/pub.go b/internal/publish/pub.go index a975493392d..fdfac0be0a7 100644 --- a/internal/publish/pub.go +++ b/internal/publish/pub.go @@ -198,7 +198,7 @@ func (t batchTransformer) Transform(context.Context) []beat.Event { if err := modeljson.MarshalAPMEvent(event, &w); err != nil { continue } - beatEvent := beat.Event{Timestamp: event.Timestamp.AsTime()} + beatEvent := beat.Event{Timestamp: modelpb.ToTime(event.Timestamp)} if err := json.Unmarshal(w.Bytes(), &beatEvent.Fields); err != nil { continue } diff --git a/internal/publish/pub_test.go b/internal/publish/pub_test.go index a0cf5aec08e..a22389f84a5 100644 --- a/internal/publish/pub_test.go +++ b/internal/publish/pub_test.go @@ -29,7 +29,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/timestamppb" "go.elastic.co/apm/v2/apmtest" @@ -154,7 +153,7 @@ func BenchmarkPublisher(b *testing.B) { batch := modelpb.Batch{ &modelpb.APMEvent{ - Timestamp: timestamppb.Now(), + Timestamp: modelpb.FromTime(time.Now()), }, } ctx := context.Background() diff --git a/systemtest/approvals/TestCompressedSpans.approved.json b/systemtest/approvals/TestCompressedSpans.approved.json index 79ef04894ca..835105e7b17 100644 --- a/systemtest/approvals/TestCompressedSpans.approved.json +++ b/systemtest/approvals/TestCompressedSpans.approved.json @@ -83,9 +83,6 @@ "subtype": "redis", "type": "db" }, - "timestamp": { - "us": 0 - }, "trace": { "id": "01000000000000000000000000000000" }, diff --git a/systemtest/approvals/TestDropUnsampled.approved.json b/systemtest/approvals/TestDropUnsampled.approved.json index b66b9b816a7..a1ec31f8994 100644 --- a/systemtest/approvals/TestDropUnsampled.approved.json +++ b/systemtest/approvals/TestDropUnsampled.approved.json @@ -57,9 +57,6 @@ "span": { "id": "dynamic" }, - "timestamp": { - "us": "dynamic" - }, "trace": { "id": "dynamic" }, diff --git a/testdata/intake-v2/spans.ndjson b/testdata/intake-v2/spans.ndjson index fc7c4efa006..dd47144da37 100644 --- a/testdata/intake-v2/spans.ndjson +++ b/testdata/intake-v2/spans.ndjson @@ -3,9 +3,9 @@ {"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "0000000011111111", "id": "1234abcdef567895", "transaction_id": "ab45781d265894fe", "name": "GET /api/types", "type": "request", "start": 22, "duration": 32.592981, "timestamp": 1532976822281000,"context":{"service":{"environment":"prod","agent":{}}}}} {"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdefabcdef7890", "id": "0123456a89012345", "transaction_id": "ab23456a89012345", "name": "GET /api/types", "type": "request.http", "start": 1.845, "duration": 3.5642981, "timestamp": 1532976822281000, "stacktrace": [], "context":{"tags": {"tag1": "value1", "tag2": 123, "tag3": 12.34, "tag4": true, "tag5": null},"service":{}}}} {"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "ababcdcdefefabde", "id": "abcde56a89012345", "transaction_id": null, "name": "get /api/types", "sync": false, "type": "request", "subtype": "http", "action": "call", "start": 0, "duration": 13.9802981, "timestamp": 1532976822281000, "stacktrace": null, "context": null }} -{"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "1234567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "timestamp": 1532976822281000, "stacktrace": [{ "filename": "net.js", "classname": "Core.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ " ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ " var trans = this.currentTransaction", "" ], "post_context": [ " ins.currentTransaction = prev", " return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com", "rows_affected": 2}, "http": { "url": "http://localhost:8000", "status_code":200, "response":{"headers": { "content-type": null }, "status_code":200,"transfer_size":30012,"encoded_body_size":356,"decoded_body_size":401}, "method": "GET" }, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgres/testdb"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"}}}}} -{"span": {"trace_id": "fdceas0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "fdc4567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "timestamp": 1532976822281000, "stacktrace": [{ "filename": "net.js", "classname": "Core.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ " ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ " var trans = this.currentTransaction", "" ], "post_context": [ " ins.currentTransaction = prev", " return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com", "rows_affected": 2}, "http": { "url": "http://localhost:8000", "status_code":200, "response":{"headers": { "content-type": null }, "status_code":200,"transfer_size":30012,"encoded_body_size":356,"decoded_body_size":401}, "method": "GET" }, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgresql"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"},"target":{"name":"testdb","type":"postgres"}}}}} -{"span": {"trace_id": "utedif0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "ute4567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "timestamp": 1532976822281000, "stacktrace": [{ "filename": "net.js", "classname": "Core.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ " ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ " var trans = this.currentTransaction", "" ], "post_context": [ " ins.currentTransaction = prev", " return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com", "rows_affected": 2}, "http": { "url": "http://localhost:8000", "status_code":200, "response":{"headers": { "content-type": null }, "status_code":200,"transfer_size":30012,"encoded_body_size":356,"decoded_body_size":401}, "method": "GET", "request": {"id": "some-request-id"}}, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgresql"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"}}}}} +{"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "1234567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "timestamp": 1532976822281000, "stacktrace": [{ "filename": "net.js", "classname": "Core.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ " ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ " var trans = this.currentTransaction", "" ], "post_context": [ " ins.currentTransaction = prev", " return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com", "rows_affected": 2}, "http": { "url": "http://localhost:8000", "status_code":200, "response":{"headers": { "content-type": [] }, "status_code":200,"transfer_size":30012,"encoded_body_size":356,"decoded_body_size":401}, "method": "GET" }, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgres/testdb"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"}}}}} +{"span": {"trace_id": "fdceas0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "fdc4567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "timestamp": 1532976822281000, "stacktrace": [{ "filename": "net.js", "classname": "Core.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ " ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ " var trans = this.currentTransaction", "" ], "post_context": [ " ins.currentTransaction = prev", " return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com", "rows_affected": 2}, "http": { "url": "http://localhost:8000", "status_code":200, "response":{"headers": { "content-type": [] }, "status_code":200,"transfer_size":30012,"encoded_body_size":356,"decoded_body_size":401}, "method": "GET" }, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgresql"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"},"target":{"name":"testdb","type":"postgres"}}}}} +{"span": {"trace_id": "utedif0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "ute4567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "timestamp": 1532976822281000, "stacktrace": [{ "filename": "net.js", "classname": "Core.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ " ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ " var trans = this.currentTransaction", "" ], "post_context": [ " ins.currentTransaction = prev", " return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com", "rows_affected": 2}, "http": { "url": "http://localhost:8000", "status_code":200, "response":{"headers": { "content-type": [] }, "status_code":200,"transfer_size":30012,"encoded_body_size":356,"decoded_body_size":401}, "method": "GET", "request": {"id": "some-request-id"}}, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgresql"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"}}}}} {"span": {"trace_id": "ljfjev0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "ljf4567890aaaade", "transaction_id": "01af25874dec69dd", "name": "Request", "type": "external", "duration":0.988, "timestamp":1532975583584217,"action": "request","outcome": "success", "stacktrace": [], "subtype": "http", "context": {"destination": {"address": "0:0::0:1", "port": 8080, "service": {"type": "http", "name": "external", "resource": "localhost:8080"}}}}} {"span": {"trace_id": "fdedef0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "00xxx12312312312", "transaction_id": "01af25874dec69dd", "name": "Rabbitmq receive", "type": "messaging", "subtype": "JMS", "action": "receive", "duration": 141.581, "timestamp": 1532976822281000, "context": {"destination": {"address": "0:0::0:1"}, "message": {"queue": { "name": "new_users"}, "age":{"ms": 1577958057123}, "routing_key": "user-created-span"}}}} {"span": {"trace_id": "edcbaf0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "abcdef01234567", "transaction_id": "01af25874dec69dd", "name": "SELECT FROM p_details", "type": "db.postgresql.query","start": 2.83092, "duration": 378.1912, "timestamp": 1625572685682272,"composite":{"count":10, "sum": 359.2981,"compression_strategy": "exact_match"},"outcome": "success"}} diff --git a/testdata/intake-v2/transactions.ndjson b/testdata/intake-v2/transactions.ndjson index 2e74d513466..0a3fa568d8b 100644 --- a/testdata/intake-v2/transactions.ndjson +++ b/testdata/intake-v2/transactions.ndjson @@ -1,6 +1,6 @@ {"metadata": {"service": {"name": "1234_service-12a3","node": {"configured_name": "node-123"},"version": "5.1.3","environment": "staging","language": {"name": "ecmascript","version": "8"},"runtime": {"name": "node","version": "8.0.0"},"framework": {"name": "Express","version": "1.2.3"},"agent": {"name": "elastic-node", "activation_method": "some_activation_method","version": "3.14.0"}},"user": {"id": "123user", "username": "bar", "email": "bar@user.com"}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "process": {"pid": 1234,"ppid": 6789,"title": "node","argv": ["node","server.js"]},"system": {"hostname": "prod1.example.com","architecture": "x64","platform": "darwin", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}},"cloud":{"account":{"id":"account_id","name":"account_name"},"availability_zone":"cloud_availability_zone","instance":{"id":"instance_id","name":"instance_name"},"machine":{"type":"machine_type"},"project":{"id":"project_id","name":"project_name"},"provider":"cloud_provider","region":"cloud_region","service":{"name":"lambda"}}}} {"transaction": { "id": "945254c567a5417e", "trace_id": "0123456789abcdef0123456789abcdef", "parent_id": "abcdefabcdef01234567", "type": "request", "duration": 32.592981, "timestamp": 1496170407154000, "span_count": { "started": 43 }}} {"transaction": {"id": "4340a8e0df1906ecbfa9", "trace_id": "0acd456789abcdef0123456789abcdef", "name": "GET /api/types","type": "request","duration": 32.592981,"outcome":"success", "result": "success", "timestamp": 1496170407154000, "sampled": true, "span_count": {"started": 17},"context": {"service": {"runtime": {"version": "7.0"}},"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"}, "request": {"socket": {"remote_address": "12.53.12.1","encrypted": true},"http_version": "1.1","method": "POST","url": {"protocol": "https:","full": "https://www.example.com/p/a/t/h?query=string#hash","hostname": "www.example.com","port": "8080","pathname": "/p/a/t/h","search": "?query=string","hash": "#hash","raw": "/p/a/t/h?query=string#hash"},"headers": {"user-agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36","Mozilla Chrome Edge"],"content-type": "text/html","cookie": "c1=v1, c2=v2","some-other-header": "foo","array": ["foo","bar","baz"]},"cookies": {"c1": "v1","c2": "v2"},"env": {"SERVER_SOFTWARE": "nginx","GATEWAY_INTERFACE": "CGI/1.1"},"body": {"str": "hello world","additional": { "foo": {},"bar": 123,"req": "additional information"}}},"response": {"status_code": 200,"headers": {"content-type": "application/json"},"headers_sent": true,"finished": true,"transfer_size":258,"encoded_body_size":2690,"decoded_body_size":2990}, "user": {"domain": "ldap://abc","id": "99","username": "foo"},"tags": {"organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8", "tag2": 12, "tag3": 12.45, "tag4": false, "tag5": null },"custom": {"my_key": 1,"some_other_value": "foo bar","and_objects": {"foo": ["bar","baz"]},"(": "not a valid regex and that is fine"}}}} -{"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": null, "content-type": null, "cookie": null }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": null } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} +{"transaction": { "id": "cdef4340a8e0df19", "trace_id": "0acd456789abcdef0123456789abcdef", "type": "request", "duration": 13.980558, "timestamp": 1532976822281000, "sampled": null, "span_count": { "dropped": 55, "started": 436 }, "marks": {"navigationTiming": {"appBeforeBootstrap": 608.9300000000001,"navigationStart": -21},"another_mark": {"some_long": 10,"some_float": 10.0}, "performance": {}}, "context": { "request": { "socket": { "remote_address": "192.0.1", "encrypted": null }, "method": "POST", "headers": { "user-agent": [], "content-type": [], "cookie": [] }, "url": { "protocol": null, "full": null, "hostname": null, "port": null, "pathname": null, "search": null, "hash": null, "raw": null } }, "response": { "headers": { "content-type": [] } }, "service": {"environment":"testing","name": "service1","node": {"configured_name": "node-ABC"}, "language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.2", "name": "elastic-ruby", "ephemeral_id": "justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}},"experience":{"cls":1,"fid":2.0,"tbt":3.4,"longtask":{"count":3,"sum":2.5,"max":1}}}} {"transaction": { "id": "00xxxxFFaaaa1234", "trace_id": "0123456789abcdef0123456789abcdef", "name": "amqp receive", "parent_id": "abcdefabcdef01234567", "type": "messaging", "duration": 3, "timestamp": 1496170407154000, "span_count": { "started": 1 }, "context": {"message": {"queue": { "name": "new_users"}, "age":{ "ms": 1577958057123}, "headers": {"user_id": "1ax3", "involved_services": ["user", "auth"]}, "body": "user created", "routing_key": "user-created-transaction"}},"session":{"id":"sunday","sequence":123}}} {"transaction": { "name": "july-2021-delete-after-july-31", "type": "lambda", "result": "success", "id": "142e61450efb8574", "trace_id": "eb56529a1f461c5e7e2f66ecb075e983", "subtype": null, "action": null, "duration": 38.853, "timestamp": 1631736666365048, "sampled": true, "context": { "cloud": { "origin": { "account": { "id": "abc123" }, "provider": "aws", "region": "us-east-1", "service": { "name": "serviceName" } } }, "service": { "origin": { "id": "abc123", "name": "service-name", "version": "1.0" } }, "user": {}, "tags": {}, "custom": { } }, "sync": true, "span_count": { "started": 0 }, "outcome": "unknown", "faas": { "name": "faasName", "version": "1.0.0", "coldstart": false, "execution": "2e13b309-23e1-417f-8bf7-074fc96bc683", "trigger": { "request_id": "FuH2Cir_vHcEMUA=", "type": "http" } }, "sample_rate": 1 } } diff --git a/x-pack/apm-server/aggregation/lsm.go b/x-pack/apm-server/aggregation/lsm.go index eced10e6f39..7f959e689f5 100644 --- a/x-pack/apm-server/aggregation/lsm.go +++ b/x-pack/apm-server/aggregation/lsm.go @@ -38,7 +38,6 @@ func New( MaxTransactionGroupsPerService: max(maxTxGroups/10, 1), MaxServiceTransactionGroups: maxSvcTxGroups, MaxServiceTransactionGroupsPerService: max(maxSvcTxGroups/10, 1), - MaxServiceInstanceGroupsPerService: max(maxSvcs/10, 1), MaxServices: maxSvcs, }), aggregators.WithProcessor(wrapNextProcessor(nextProcessor)), diff --git a/x-pack/apm-server/sampling/groups.go b/x-pack/apm-server/sampling/groups.go index fd0177f476f..f399b331f9a 100644 --- a/x-pack/apm-server/sampling/groups.go +++ b/x-pack/apm-server/sampling/groups.go @@ -164,7 +164,7 @@ func (g *traceGroup) sampleTrace(transactionEvent *modelpb.APMEvent) (bool, erro defer g.mu.Unlock() g.total++ return g.reservoir.Sample( - transactionEvent.GetEvent().GetDuration().AsDuration().Seconds(), + time.Duration(transactionEvent.GetEvent().GetDuration()).Seconds(), transactionEvent.GetTrace().GetId(), ), nil } diff --git a/x-pack/apm-server/sampling/groups_test.go b/x-pack/apm-server/sampling/groups_test.go index ff347901c34..446bdf54c03 100644 --- a/x-pack/apm-server/sampling/groups_test.go +++ b/x-pack/apm-server/sampling/groups_test.go @@ -12,7 +12,6 @@ import ( "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/durationpb" "github.com/elastic/apm-data/model/modelpb" ) @@ -272,7 +271,7 @@ func BenchmarkTraceGroups(b *testing.B) { // Duration is non-zero to ensure transactions have a non-zero chance of // being sampled. tx := modelpb.APMEvent{ - Event: &modelpb.Event{Duration: durationpb.New(time.Second)}, + Event: &modelpb.Event{Duration: uint64(time.Second)}, Transaction: &modelpb.Transaction{ Type: "type", Name: uuid.Must(uuid.NewV4()).String(), @@ -280,7 +279,7 @@ func BenchmarkTraceGroups(b *testing.B) { } for pb.Next() { groups.sampleTrace(&tx) - tx.Event.Duration = durationpb.New(tx.Event.Duration.AsDuration() + time.Second) + tx.Event.Duration = tx.Event.Duration + uint64(time.Second) } }) } diff --git a/x-pack/apm-server/sampling/processor_test.go b/x-pack/apm-server/sampling/processor_test.go index 70eb8cdda6d..5df33f19f77 100644 --- a/x-pack/apm-server/sampling/processor_test.go +++ b/x-pack/apm-server/sampling/processor_test.go @@ -23,7 +23,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" - "google.golang.org/protobuf/types/known/durationpb" "github.com/elastic/apm-data/model/modelpb" "github.com/elastic/apm-server/x-pack/apm-server/sampling" @@ -170,7 +169,7 @@ func TestProcessLocalTailSampling(t *testing.T) { trace2 := modelpb.Trace{Id: "0102030405060708090a0b0c0d0e0f11"} trace1Events := modelpb.Batch{{ Trace: &trace1, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Transaction: &modelpb.Transaction{ Type: "type", Id: "0102030405060708", @@ -178,7 +177,7 @@ func TestProcessLocalTailSampling(t *testing.T) { }, }, { Trace: &trace1, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Span: &modelpb.Span{ Type: "type", Id: "0102030405060709", @@ -186,7 +185,7 @@ func TestProcessLocalTailSampling(t *testing.T) { }} trace2Events := modelpb.Batch{{ Trace: &trace2, - Event: &modelpb.Event{Duration: durationpb.New(456 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(456 * time.Millisecond)}, Transaction: &modelpb.Transaction{ Type: "type", Id: "0102030405060710", @@ -194,7 +193,7 @@ func TestProcessLocalTailSampling(t *testing.T) { }, }, { Trace: &trace2, - Event: &modelpb.Event{Duration: durationpb.New(456 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(456 * time.Millisecond)}, Span: &modelpb.Span{ Type: "type", Id: "0102030405060711", @@ -291,7 +290,7 @@ func TestProcessLocalTailSamplingUnsampled(t *testing.T) { traceIDs[i] = traceID batch := modelpb.Batch{{ Trace: &modelpb.Trace{Id: traceID}, - Event: &modelpb.Event{Duration: durationpb.New(time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(time.Millisecond)}, Transaction: &modelpb.Transaction{ Type: "type", Id: traceID, @@ -363,7 +362,7 @@ func TestProcessLocalTailSamplingPolicyOrder(t *testing.T) { events[i] = &modelpb.APMEvent{ Service: &service, Trace: &modelpb.Trace{Id: fmt.Sprintf("%x", traceIDBytes[:])}, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Transaction: &modelpb.Transaction{ Type: "type", Name: "trace_name", @@ -432,7 +431,7 @@ func TestProcessRemoteTailSampling(t *testing.T) { traceID2 := "0102030405060708090a0b0c0d0e0f11" trace1Events := modelpb.Batch{{ Trace: &modelpb.Trace{Id: traceID1}, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Span: &modelpb.Span{ Type: "type", Id: "0102030405060709", @@ -517,7 +516,7 @@ func TestGroupsMonitoring(t *testing.T) { err := processor.ProcessBatch(context.Background(), &modelpb.Batch{{ Service: &modelpb.Service{Name: fmt.Sprintf("service_%d", i)}, Trace: &modelpb.Trace{Id: uuid.Must(uuid.NewV4()).String()}, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Transaction: &modelpb.Transaction{ Type: "type", Id: "0102030405060709", @@ -549,7 +548,7 @@ func TestStorageMonitoring(t *testing.T) { traceID := uuid.Must(uuid.NewV4()).String() batch := modelpb.Batch{{ Trace: &modelpb.Trace{Id: traceID}, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Transaction: &modelpb.Transaction{ Type: "type", Id: traceID, @@ -603,7 +602,7 @@ func TestStorageGC(t *testing.T) { traceID := uuid.Must(uuid.NewV4()).String() batch := modelpb.Batch{{ Trace: &modelpb.Trace{Id: traceID}, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Span: &modelpb.Span{ Type: "type", Id: traceID, @@ -674,7 +673,7 @@ func TestStorageLimit(t *testing.T) { traceID := uuid.Must(uuid.NewV4()).String() batch = append(batch, &modelpb.APMEvent{ Trace: &modelpb.Trace{Id: traceID}, - Event: &modelpb.Event{Duration: durationpb.New(123 * time.Millisecond)}, + Event: &modelpb.Event{Duration: uint64(123 * time.Millisecond)}, Span: &modelpb.Span{ Type: "type", Id: traceID, diff --git a/x-pack/apm-server/sampling/pubsub/pubsub.go b/x-pack/apm-server/sampling/pubsub/pubsub.go index ed6886ab113..a3cecc45d8a 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub.go @@ -19,7 +19,6 @@ import ( "go.elastic.co/fastjson" "go.uber.org/zap" "golang.org/x/sync/errgroup" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-docappender" @@ -101,7 +100,7 @@ func (p *Pubsub) indexSampledTraceIDs(ctx context.Context, traceIDs <-chan strin } var w fastjson.Writer doc := modelpb.APMEvent{ - Timestamp: timestamppb.New(time.Now()), + Timestamp: modelpb.FromTime(time.Now()), DataStream: &modelpb.DataStream{ Type: p.config.DataStream.Type, Dataset: p.config.DataStream.Dataset,