Add data quality metric to measure traces without a root (#3812)

* Add data quality metric to measure traces without a root * chlog
grafana · Jun 27, 2024 · fb4deaf · fb4deaf
1 parent b3f06d4
commit fb4deaf
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,7 @@
 * [ENHANCEMENT] Protect ingesters from panics by adding defer/recover to all read path methods. [#3790](https://github.com/grafana/tempo/pull/3790) (@joe-elliott)
 * [ENHANCEMENT] Added a boolean flag to enable or disable dualstack mode on Storage block config for S3 [#3721](https://github.com/grafana/tempo/pull/3721) (@sid-jar, @mapno)
 * [ENHANCEMENT] Add caching to query range queries [#3796](https://github.com/grafana/tempo/pull/3796) (@mapno)
+* [ENHANCEMENT] Add data quality metric to measure traces without a root [#3812](https://github.com/grafana/tempo/pull/3812) (@mapno)
 * [BUGFIX] Fix metrics queries when grouping by attributes that may not exist [#3734](https://github.com/grafana/tempo/pull/3734) (@mdisibio)
 * [BUGFIX] Fix frontend parsing error on cached responses [#3759](https://github.com/grafana/tempo/pull/3759) (@mdisibio)
 * [BUGFIX] max_global_traces_per_user: take into account ingestion.tenant_shard_size when converting to local limit [#3618](https://github.com/grafana/tempo/pull/3618) (@kvrhdn)

diff --git a/pkg/dataquality/warnings.go b/pkg/dataquality/warnings.go
@@ -8,6 +8,7 @@ import (
 const (
 	reasonOutsideIngestionSlack = "outside_ingestion_time_slack"
 	reasonDisconnectedTrace     = "disconnected_trace"
+	reasonRootlessTrace         = "rootless_trace"
 
 	PhaseTraceFlushedToWal     = "_flushed_to_wal"
 	PhaseTraceWalToComplete    = "_wal_to_complete"
@@ -27,3 +28,7 @@ func WarnOutsideIngestionSlack(tenant string) {
 func WarnDisconnectedTrace(tenant string, phase string) {
 	metric.WithLabelValues(tenant, reasonDisconnectedTrace+phase).Inc()
 }
+
+func WarnRootlessTrace(tenant string, phase string) {
+	metric.WithLabelValues(tenant, reasonRootlessTrace+phase).Inc()
+}
diff --git a/tempodb/compactor.go b/tempodb/compactor.go
@@ -236,6 +236,9 @@ func (rw *readerWriter) compact(ctx context.Context, blockMetas []*backend.Block
 		DisconnectedTrace: func() {
 			dataquality.WarnDisconnectedTrace(tenantID, dataquality.PhaseTraceCompactorCombine)
 		},
+		RootlessTrace: func() {
+			dataquality.WarnRootlessTrace(tenantID, dataquality.PhaseTraceCompactorCombine)
+		},
 	}
 
 	compactor := enc.NewCompactor(opts)

diff --git a/tempodb/encoding/common/interfaces.go b/tempodb/encoding/common/interfaces.go
@@ -80,6 +80,7 @@ type CompactionOptions struct {
 	BytesWritten      func(compactionLevel, bytes int)
 	SpansDiscarded    func(traceID string, rootSpanName string, rootServiceName string, spans int)
 	DisconnectedTrace func()
+	RootlessTrace     func()
 }
 
 type Iterator interface {

diff --git a/tempodb/encoding/vparquet3/compactor.go b/tempodb/encoding/vparquet3/compactor.go
@@ -124,6 +124,9 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader,
 		if !connected {
 			c.opts.DisconnectedTrace()
 		}
+		if tr != nil && tr.RootSpanName == "" {
+			c.opts.RootlessTrace()
+		}
 
 		c.opts.ObjectsCombined(int(compactionLevel), 1)
 		return sch.Deconstruct(pool.Get(), tr), nil

diff --git a/tempodb/encoding/vparquet3/wal_block.go b/tempodb/encoding/vparquet3/wal_block.go
@@ -335,6 +335,9 @@ func (b *walBlock) AppendTrace(id common.ID, trace *tempopb.Trace, start, end ui
 	if !connected {
 		dataquality.WarnDisconnectedTrace(b.meta.TenantID, dataquality.PhaseTraceFlushedToWal)
 	}
+	if b.buffer != nil && b.buffer.RootSpanName == "" {
+		dataquality.WarnRootlessTrace(b.meta.TenantID, dataquality.PhaseTraceFlushedToWal)
+	}
 
 	start, end = b.adjustTimeRangeForSlack(start, end, 0)
 

diff --git a/tempodb/encoding/vparquet4/compactor.go b/tempodb/encoding/vparquet4/compactor.go
@@ -124,6 +124,9 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader,
 		if !connected {
 			c.opts.DisconnectedTrace()
 		}
+		if tr != nil && tr.RootSpanName == "" {
+			c.opts.RootlessTrace()
+		}
 
 		c.opts.ObjectsCombined(int(compactionLevel), 1)
 		return sch.Deconstruct(pool.Get(), tr), nil

diff --git a/tempodb/encoding/vparquet4/wal_block.go b/tempodb/encoding/vparquet4/wal_block.go
@@ -335,6 +335,9 @@ func (b *walBlock) AppendTrace(id common.ID, trace *tempopb.Trace, start, end ui
 	if !connected {
 		dataquality.WarnDisconnectedTrace(b.meta.TenantID, dataquality.PhaseTraceFlushedToWal)
 	}
+	if b.buffer != nil && b.buffer.RootSpanName == "" {
+		dataquality.WarnRootlessTrace(b.meta.TenantID, dataquality.PhaseTraceFlushedToWal)
+	}
 
 	start, end = b.adjustTimeRangeForSlack(start, end, 0)