Replace TenantIndex, BlockMeta and CompactedBlockMeta with proto near-equivalents

CHANGELOG.md

@@ -30,7 +30,7 @@
 * [ENHANCEMENT] Replace Grafana Agent example by Grafana Alloy[#4030](https://github.com/grafana/tempo/pull/4030) (@javiermolinar)
 * [ENHANCEMENT] Support exporting internal Tempo traces via OTLP exporter when `use_otel_tracer` is enabled. Use the OpenTelemetry SDK environment variables to configure the span exporter. [#4028](https://github.com/grafana/tempo/pull/4028) (@andreasgerstmayr)
 * [ENHANCEMENT] TraceQL metrics queries: add min_over_time  [#3975](https://github.com/grafana/tempo/pull/3975) (@javiermolinar)
-
+* [ENHANCEMENT] Write tenantindex as proto and json with a prefernce for proto [#4072](https://github.com/grafana/tempo/pull/4072) (@zalegrala)
 
 # v2.6.0
 

Makefile

@@ -223,6 +223,7 @@ PROTO_INTERMEDIATE_DIR = pkg/.patched-proto
 PROTO_INCLUDES = -I$(PROTO_INTERMEDIATE_DIR)
 PROTO_GEN = $(PROTOC) $(PROTO_INCLUDES) --gogofaster_out=plugins=grpc,paths=source_relative:$(2) $(1)
 PROTO_GEN_WITH_VENDOR = $(PROTOC) $(PROTO_INCLUDES) -Ivendor -Ivendor/github.com/gogo/protobuf --gogofaster_out=plugins=grpc,paths=source_relative:$(2) $(1)
+PROTO_GEN_WITHOUT_RELATIVE = $(PROTOC) $(PROTO_INCLUDES) --gogofaster_out=plugins=grpc:$(2) $(1)
 
 .PHONY: gen-proto
 gen-proto:  ## Generate proto files
@@ -264,6 +265,7 @@ gen-proto:  ## Generate proto files
 	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/resource/v1/resource.proto,./pkg/tempopb/)
 	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/trace/v1/trace.proto,./pkg/tempopb/)
 	$(call PROTO_GEN,pkg/tempopb/tempo.proto,./)
+	$(call PROTO_GEN_WITHOUT_RELATIVE,tempodb/backend/v1/v1.proto,./)
 	$(call PROTO_GEN_WITH_VENDOR,modules/frontend/v1/frontendv1pb/frontend.proto,./)
 
 	rm -rf $(PROTO_INTERMEDIATE_DIR)

cmd/tempo-cli/cmd-analyse-block.go

@@ -111,7 +111,7 @@ func (cmd *analyseBlockCmd) Run(ctx *globalOptions) error {
 	return blockSum.print(cmd.NumAttr, cmd.GenerateJsonnet, cmd.SimpleSummary, cmd.PrintFullSummary)
 }
 
-func processBlock(r backend.Reader, tenantID, blockID string, maxStartTime, minStartTime time.Time, minCompactionLvl uint8) (*blockSummary, error) {
+func processBlock(r backend.Reader, tenantID, blockID string, maxStartTime, minStartTime time.Time, minCompactionLvl uint32) (*blockSummary, error) {
 	id := uuid.MustParse(blockID)
 
 	meta, err := r.BlockMeta(context.TODO(), id, tenantID)
@@ -143,9 +143,9 @@ func processBlock(r backend.Reader, tenantID, blockID string, maxStartTime, minS
 		return nil, nil
 	}
 
-	br := tempo_io.NewBufferedReaderAt(reader, int64(meta.Size), 2*1024*1024, 64) // 128 MB memory buffering
+	br := tempo_io.NewBufferedReaderAt(reader, int64(meta.Size_), 2*1024*1024, 64) // 128 MB memory buffering
 
-	pf, err := parquet.OpenFile(br, int64(meta.Size), parquet.SkipBloomFilters(true), parquet.SkipPageIndex(true))
+	pf, err := parquet.OpenFile(br, int64(meta.Size_), parquet.SkipBloomFilters(true), parquet.SkipPageIndex(true))
 	if err != nil {
 		return nil, err
 	}

cmd/tempo-cli/cmd-analyse-blocks.go

@@ -60,7 +60,7 @@ func (cmd *analyseBlocksCmd) Run(ctx *globalOptions) error {
 			continue
 		}
 
-		blockSum, err := processBlock(r, cmd.TenantID, block.String(), maxStartTime, minStartTime, uint8(cmd.MinCompactionLevel))
+		blockSum, err := processBlock(r, cmd.TenantID, block.String(), maxStartTime, minStartTime, uint32(cmd.MinCompactionLevel))
 		if err != nil {
 			if !errors.Is(err, backend.ErrDoesNotExist) {
 				return err

cmd/tempo-cli/cmd-convert-parquet-3to4.go

@@ -104,7 +104,7 @@ func (cmd *convertParquet3to4) Run() error {
 		return err
 	}
 
-	fmt.Printf("Successfully created block with size=%d and footerSize=%d\n", outMeta.Size, outMeta.FooterSize)
+	fmt.Printf("Successfully created block with size=%d and footerSize=%d\n", outMeta.Size_, outMeta.FooterSize)
 	return nil
 }
 

cmd/tempo-cli/cmd-list-block.go

@@ -73,7 +73,7 @@ func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID s
 	fmt.Println("ID            : ", unifiedMeta.BlockID)
 	fmt.Println("Version       : ", unifiedMeta.Version)
 	fmt.Println("Total Objects : ", unifiedMeta.TotalObjects)
-	fmt.Println("Data Size     : ", humanize.Bytes(unifiedMeta.Size))
+	fmt.Println("Data Size     : ", humanize.Bytes(unifiedMeta.Size_))
 	fmt.Println("Encoding      : ", unifiedMeta.Encoding)
 	fmt.Println("Level         : ", unifiedMeta.CompactionLevel)
 	fmt.Println("Window        : ", unifiedMeta.window)

cmd/tempo-cli/cmd-list-blocks.go

@@ -56,9 +56,9 @@ func displayResults(results []blockStats, windowDuration time.Duration, includeC
 			case "lvl":
 				s = strconv.Itoa(int(r.CompactionLevel))
 			case "objects":
-				s = strconv.Itoa(r.TotalObjects)
+				s = strconv.Itoa(int(r.TotalObjects))
 			case "size":
-				s = fmt.Sprintf("%v", humanize.Bytes(r.Size))
+				s = fmt.Sprintf("%v", humanize.Bytes(r.Size_))
 			case "encoding":
 				s = r.Encoding.String()
 			case "vers":
@@ -89,8 +89,8 @@ func displayResults(results []blockStats, windowDuration time.Duration, includeC
 		}
 
 		out = append(out, line)
-		totalObjects += r.TotalObjects
-		totalBytes += r.Size
+		totalObjects += int(r.TotalObjects)
+		totalBytes += r.Size_
 	}
 
 	footer := make([]string, 0)

cmd/tempo-cli/cmd-list-column.go

@@ -33,7 +33,7 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error {
 	}
 
 	rr := vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta)
-	pf, err := parquet.OpenFile(rr, int64(meta.Size))
+	pf, err := parquet.OpenFile(rr, int64(meta.Size_))
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-list-compactionsummary.go

@@ -61,24 +61,24 @@ func displayCompactionSummary(results []blockStats) {
 		sizeSum := uint64(0)
 		sizeMin := uint64(0)
 		sizeMax := uint64(0)
-		countSum := 0
-		countMin := 0
-		countMax := 0
+		countSum := int64(0)
+		countMin := int64(0)
+		countMax := int64(0)
 		countBloomShards := 0
 
 		var newest time.Time
 		var oldest time.Time
 
 		for _, r := range resultsByLevel[l] {
-			sizeSum += r.Size
+			sizeSum += r.Size_
 			countSum += r.TotalObjects
 			countBloomShards += int(r.BloomShardCount)
 
-			if r.Size < sizeMin || sizeMin == 0 {
-				sizeMin = r.Size
+			if r.Size_ < sizeMin || sizeMin == 0 {
+				sizeMin = r.Size_
 			}
-			if r.Size > sizeMax {
-				sizeMax = r.Size
+			if r.Size_ > sizeMax {
+				sizeMax = r.Size_
 			}
 			if r.TotalObjects < countMin || countMin == 0 {
 				countMin = r.TotalObjects

cmd/tempo-cli/cmd-migrate-tenant.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/dustin/go-humanize"
+	"github.com/google/uuid"
 
 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/grafana/tempo/tempodb/encoding"
@@ -49,7 +50,7 @@ blocks:
 	for _, sourceBlockMeta := range sourceTenantIndex.Meta {
 		// check for collisions
 		for _, uuidDest := range blocksDest {
-			if sourceBlockMeta.BlockID == uuidDest {
+			if (uuid.UUID)(sourceBlockMeta.BlockID) == uuidDest {
 				fmt.Printf("UUID %s exists in source and destination, skipping block\n", sourceBlockMeta.BlockID)
 				continue blocks
 			}
@@ -70,7 +71,7 @@ blocks:
 		}
 
 		copiedBlocks++
-		copiedSize += sourceBlockMeta.Size
+		copiedSize += sourceBlockMeta.Size_
 	}
 
 	fmt.Printf("Finished migrating data. Copied %d blocks, %s\n", copiedBlocks, humanize.Bytes(copiedSize))

cmd/tempo-cli/cmd-rewrite-blocks.go

@@ -61,7 +61,7 @@ func (cmd *dropTraceCmd) Run(ctx *globalOptions) error {
 	// print out blocks that have the trace id
 	fmt.Println("\n\ntrace found in:")
 	for _, block := range blocks {
-		fmt.Printf("  %v sz: %d traces: %d\n", block.BlockID, block.Size, block.TotalObjects)
+		fmt.Printf("  %v sz: %d traces: %d\n", block.BlockID, block.Size_, block.TotalObjects)
 	}
 
 	if !cmd.DropTrace {
@@ -82,7 +82,7 @@ func (cmd *dropTraceCmd) Run(ctx *globalOptions) error {
 	fmt.Println("marking old blocks compacted")
 	for _, block := range blocks {
 		fmt.Printf("  marking %v\n", block.BlockID)
-		err = c.MarkBlockCompacted(block.BlockID, block.TenantID)
+		err = c.MarkBlockCompacted((uuid.UUID)(block.BlockID), block.TenantID)
 		if err != nil {
 			return err
 		}

cmd/tempo-cli/cmd-view-pq-schema.go

@@ -37,7 +37,7 @@ func (cmd *viewSchemaCmd) Run(ctx *globalOptions) error {
 	fmt.Printf("%+v\n", meta)
 
 	rr := vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta)
-	pf, err := parquet.OpenFile(rr, int64(meta.Size))
+	pf, err := parquet.OpenFile(rr, int64(meta.Size_))
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/shared.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-
 	"github.com/grafana/tempo/pkg/boundedwaitgroup"
 	"github.com/grafana/tempo/tempodb/backend"
 )
@@ -42,7 +41,7 @@ func getMeta(meta *backend.BlockMeta, compactedMeta *backend.CompactedBlockMeta,
 
 	return unifiedBlockMeta{
 		BlockMeta: backend.BlockMeta{
-			BlockID:         uuid.UUID{},
+			BlockID:         backend.UUID{},
 			CompactionLevel: 0,
 			TotalObjects:    -1,
 		},
@@ -72,7 +71,7 @@ func loadBucket(r backend.Reader, c backend.Compactor, tenantID string, windowRa
 	for blockNum, id := range blockIDs {
 		wg.Add(1)
 
-		go func(id2 uuid.UUID, blockNum2 int) {
+		go func(id2 backend.UUID, blockNum2 int) {
 			defer wg.Done()
 
 			b, err := loadBlock(r, c, tenantID, id2, blockNum2, windowRange, includeCompacted)
@@ -84,7 +83,7 @@ func loadBucket(r backend.Reader, c backend.Compactor, tenantID string, windowRa
 			if b != nil {
 				resultsCh <- *b
 			}
-		}(id, blockNum)
+		}(backend.UUID(id), blockNum)
 	}
 
 	wg.Wait()
@@ -102,20 +101,20 @@ func loadBucket(r backend.Reader, c backend.Compactor, tenantID string, windowRa
 	return results, nil
 }
 
-func loadBlock(r backend.Reader, c backend.Compactor, tenantID string, id uuid.UUID, blockNum int, windowRange time.Duration, includeCompacted bool) (*blockStats, error) {
+func loadBlock(r backend.Reader, c backend.Compactor, tenantID string, id backend.UUID, blockNum int, windowRange time.Duration, includeCompacted bool) (*blockStats, error) {
 	fmt.Print(".")
 	if blockNum%100 == 0 {
 		fmt.Print(strconv.Itoa(blockNum))
 	}
 
-	meta, err := r.BlockMeta(context.Background(), id, tenantID)
+	meta, err := r.BlockMeta(context.Background(), (uuid.UUID)(id), tenantID)
 	if errors.Is(err, backend.ErrDoesNotExist) && !includeCompacted {
 		return nil, nil
 	} else if err != nil && !errors.Is(err, backend.ErrDoesNotExist) {
 		return nil, err
 	}
 
-	compactedMeta, err := c.CompactedBlockMeta(id, tenantID)
+	compactedMeta, err := c.CompactedBlockMeta((uuid.UUID)(id), tenantID)
 	if err != nil && !errors.Is(err, backend.ErrDoesNotExist) {
 		return nil, err
 	}

cmd/tempo-serverless/handler.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/google/uuid"
 	"github.com/grafana/dskit/flagext"
 	"github.com/grafana/dskit/user"
 	"github.com/mitchellh/mapstructure"
@@ -71,7 +70,7 @@ func Handler(r *http.Request) (*tempopb.SearchResponse, *HTTPError) {
 		return nil, httpError("extracting org id", err, http.StatusBadRequest)
 	}
 
-	blockID, err := uuid.Parse(searchReq.BlockID)
+	blockID, err := backend.ParseUUID(searchReq.BlockID)
 	if err != nil {
 		return nil, httpError("parsing uuid", err, http.StatusBadRequest)
 	}
@@ -95,7 +94,7 @@ func Handler(r *http.Request) (*tempopb.SearchResponse, *HTTPError) {
 		TotalRecords:     searchReq.TotalRecords,
 		BlockID:          blockID,
 		DataEncoding:     searchReq.DataEncoding,
-		Size:             searchReq.Size_,
+		Size_:            searchReq.Size_,
 		FooterSize:       searchReq.FooterSize,
 		DedicatedColumns: dc,
 	}

integration/poller/poller_test.go

@@ -190,7 +190,7 @@ func TestPollerOwnership(t *testing.T) {
 
 					actual := []uuid.UUID{}
 					for _, m := range metas {
-						actual = append(actual, m.BlockID)
+						actual = append(actual, (uuid.UUID)(m.BlockID))
 					}
 
 					sort.Slice(actual, func(i, j int) bool { return actual[i].String() < actual[j].String() })
@@ -371,7 +371,7 @@ func TestTenantDeletion(t *testing.T) {
 
 func found(id uuid.UUID, blockMetas []*backend.BlockMeta) bool {
 	for _, b := range blockMetas {
-		if b.BlockID == id {
+		if (uuid.UUID)(b.BlockID) == id {
 			return true
 		}
 	}
@@ -383,7 +383,7 @@ func writeTenantBlocks(t *testing.T, w backend.Writer, tenant string, blockIDs [
 	var err error
 	for _, b := range blockIDs {
 		meta := &backend.BlockMeta{
-			BlockID:  b,
+			BlockID:  backend.UUID(b),
 			TenantID: tenant,
 		}
 

modules/frontend/cache_keys_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/grafana/tempo/pkg/tempopb"
 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/stretchr/testify/require"
@@ -31,7 +30,7 @@ func TestCacheKeyForJob(t *testing.T) {
 				End:   20,
 			},
 			meta: &backend.BlockMeta{
-				BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+				BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 				StartTime: time.Unix(15, 0),
 				EndTime:   time.Unix(16, 0),
 			},
@@ -47,7 +46,7 @@ func TestCacheKeyForJob(t *testing.T) {
 				End:   20,
 			},
 			meta: &backend.BlockMeta{
-				BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+				BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 				StartTime: time.Unix(15, 0),
 				EndTime:   time.Unix(16, 0),
 			},
@@ -63,7 +62,7 @@ func TestCacheKeyForJob(t *testing.T) {
 				End:   20,
 			},
 			meta: &backend.BlockMeta{
-				BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+				BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 				StartTime: time.Unix(5, 0),
 				EndTime:   time.Unix(6, 0),
 			},
@@ -79,7 +78,7 @@ func TestCacheKeyForJob(t *testing.T) {
 				End:   20,
 			},
 			meta: &backend.BlockMeta{
-				BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+				BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 				StartTime: time.Unix(5, 0),
 				EndTime:   time.Unix(15, 0),
 			},
@@ -95,7 +94,7 @@ func TestCacheKeyForJob(t *testing.T) {
 				End:   20,
 			},
 			meta: &backend.BlockMeta{
-				BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+				BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 				StartTime: time.Unix(15, 0),
 				EndTime:   time.Unix(25, 0),
 			},
@@ -111,7 +110,7 @@ func TestCacheKeyForJob(t *testing.T) {
 				End:   20,
 			},
 			meta: &backend.BlockMeta{
-				BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+				BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 				StartTime: time.Unix(25, 0),
 				EndTime:   time.Unix(30, 0),
 			},
@@ -127,7 +126,7 @@ func TestCacheKeyForJob(t *testing.T) {
 				End:   20,
 			},
 			meta: &backend.BlockMeta{
-				BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+				BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 				StartTime: time.Unix(5, 0),
 				EndTime:   time.Unix(30, 0),
 			},
@@ -151,7 +150,7 @@ func BenchmarkCacheKeyForJob(b *testing.B) {
 		End:   20,
 	}
 	meta := &backend.BlockMeta{
-		BlockID:   uuid.MustParse("00000000-0000-0000-0000-000000000123"),
+		BlockID:   backend.MustParse("00000000-0000-0000-0000-000000000123"),
 		StartTime: time.Unix(15, 0),
 		EndTime:   time.Unix(16, 0),
 	}