From db482a6d31a7ef68bedc7b8374d962e20a4c9011 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 14 Aug 2024 15:33:57 -0400 Subject: [PATCH] Clone the diff in GRPC streaming to prevent panics (#3961) * clone diff Signed-off-by: Joe Elliott * changelog Signed-off-by: Joe Elliott * add similar logic to final Signed-off-by: Joe Elliott --------- Signed-off-by: Joe Elliott (cherry picked from commit 89b8d7d783e7e4c8b2955bd3b39c9e084f75ca4c) --- CHANGELOG.md | 2 ++ modules/frontend/combiner/common.go | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea9ceda1837..e98870dafcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,8 @@ * [BUGFIX] Fix prefix handling in Azure backend Find() call [#3875](https://github.com/grafana/tempo/pull/3875) (@zalegrala) * [BUGFIX] **BREAKING CHANGE** Remove unused properties from the WAL configuration [#3911](https://github.com/grafana/tempo/pull/3911) (@javiermolinar) * [BUGFIX] Bring back OTEL receiver metrics [#3917](https://github.com/grafana/tempo/pull/3917) (@javiermolinar) +* [BUGFIX] Correct block end time when the ingested traces are outside the ingestion slack [#3954](https://github.com/grafana/tempo/pull/3954) (@javiermolinar) +* [BUGFIX] Fix race condition where a streaming response could be marshalled while being modified in the combiner resulting in a panic. [#3961](https://github.com/grafana/tempo/pull/3961) (@joe-elliott) ## v2.5.0 diff --git a/modules/frontend/combiner/common.go b/modules/frontend/combiner/common.go index c4b7d0f803c..87fffe2cd9b 100644 --- a/modules/frontend/combiner/common.go +++ b/modules/frontend/combiner/common.go @@ -176,7 +176,9 @@ func (c *genericCombiner[T]) GRPCFinal() (T, error) { return empty, err } - return final, nil + // clone the final response to prevent race conditions with marshalling this data + finalClone := proto.Clone(final).(T) + return finalClone, nil } func (c *genericCombiner[T]) GRPCDiff() (T, error) { @@ -194,7 +196,9 @@ func (c *genericCombiner[T]) GRPCDiff() (T, error) { return empty, err } - return diff, nil + // clone the diff to prevent race conditions with marshalling this data + diffClone := proto.Clone(diff) + return diffClone.(T), nil } func (c *genericCombiner[T]) erroredResponse() (*http.Response, error) {