enhance: Remove segment-level tag from monitoring metrics (#37696)

When there are a large number of segments, the metrics consume a lot of memory. This PR Remove segment-level tag from monitoring metrics. issue: #37636 Signed-off-by: bigsheeper <[email protected]>
milvus-io · Nov 16, 2024 · 3df2c92 · 3df2c92
1 parent f7c7ac5
commit 3df2c92
Show file tree

Hide file tree

Showing 5 changed files with 3 additions and 32 deletions.
diff --git a/internal/datacoord/index_meta.go b/internal/datacoord/index_meta.go
@@ -883,7 +883,7 @@ func (m *indexMeta) SetStoredIndexFileSizeMetric(collections map[UniqueID]*colle
 		coll, ok := collections[segmentIdx.CollectionID]
 		if ok {
 			metrics.DataCoordStoredIndexFilesSize.WithLabelValues(coll.DatabaseName, coll.Schema.GetName(),
-				fmt.Sprint(segmentIdx.CollectionID), fmt.Sprint(segmentIdx.SegmentID)).Set(float64(segmentIdx.IndexSize))
+				fmt.Sprint(segmentIdx.CollectionID)).Set(float64(segmentIdx.IndexSize))
 			total += segmentIdx.IndexSize
 		}
 	}

diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go
@@ -425,7 +425,7 @@ func (m *meta) GetQuotaInfo() *metricsinfo.DataCoordQuotaMetrics {
 			coll, ok := m.collections[segment.GetCollectionID()]
 			if ok {
 				metrics.DataCoordStoredBinlogSize.WithLabelValues(coll.DatabaseName,
-					fmt.Sprint(segment.GetCollectionID()), fmt.Sprint(segment.GetID()), segment.GetState().String()).Set(float64(segmentSize))
+					fmt.Sprint(segment.GetCollectionID()), segment.GetState().String()).Set(float64(segmentSize))
 			} else {
 				log.Warn("not found database name", zap.Int64("collectionID", segment.GetCollectionID()))
 			}
@@ -524,10 +524,6 @@ func (m *meta) DropSegment(segmentID UniqueID) error {
 		return err
 	}
 	metrics.DataCoordNumSegments.WithLabelValues(segment.GetState().String(), segment.GetLevel().String(), getSortStatus(segment.GetIsSorted())).Dec()
-	coll, ok := m.collections[segment.CollectionID]
-	if ok {
-		metrics.CleanupDataCoordSegmentMetrics(coll.DatabaseName, segment.CollectionID, segment.ID)
-	}
 
 	m.segments.DropSegment(segmentID)
 	log.Info("meta update: dropping segment - complete",

diff --git a/internal/metastore/kv/datacoord/kv_catalog.go b/internal/metastore/kv/datacoord/kv_catalog.go
@@ -375,9 +375,7 @@ func (kc *Catalog) collectMetrics(s *datapb.SegmentInfo) {
 	cnt += statsFieldFn(s.GetStatslogs())
 	cnt += statsFieldFn(s.GetDeltalogs())
 
-	metrics.DataCoordSegmentBinLogFileCount.
-		WithLabelValues(fmt.Sprint(s.CollectionID), fmt.Sprint(s.GetID())).
-		Set(float64(cnt))
+	metrics.DataCoordSegmentBinLogFileCount.WithLabelValues(fmt.Sprint(s.CollectionID)).Set(float64(cnt))
 }
 
 func (kc *Catalog) hasBinlogPrefix(segment *datapb.SegmentInfo) (bool, error) {

diff --git a/pkg/metrics/datacoord_metrics.go b/pkg/metrics/datacoord_metrics.go
@@ -142,7 +142,6 @@ var (
 		}, []string{
 			databaseLabelName,
 			collectionIDLabelName,
-			segmentIDLabelName,
 			segmentStateLabelName,
 		})
 	DataCoordSegmentBinLogFileCount = prometheus.NewGaugeVec(
@@ -153,7 +152,6 @@ var (
 			Help:      "number of binlog files for each segment",
 		}, []string{
 			collectionIDLabelName,
-			segmentIDLabelName,
 		})
 
 	DataCoordStoredIndexFilesSize = prometheus.NewGaugeVec(
@@ -166,7 +164,6 @@ var (
 			databaseLabelName,
 			collectionName,
 			collectionIDLabelName,
-			segmentIDLabelName,
 		})
 
 	DataCoordDmlChannelNum = prometheus.NewGaugeVec(
@@ -393,25 +390,6 @@ func RegisterDataCoord(registry *prometheus.Registry) {
 	registerStreamingCoord(registry)
 }
 
-func CleanupDataCoordSegmentMetrics(dbName string, collectionID int64, segmentID int64) {
-	DataCoordSegmentBinLogFileCount.
-		Delete(
-			prometheus.Labels{
-				collectionIDLabelName: fmt.Sprint(collectionID),
-				segmentIDLabelName:    fmt.Sprint(segmentID),
-			})
-	DataCoordStoredBinlogSize.Delete(prometheus.Labels{
-		databaseLabelName:     dbName,
-		collectionIDLabelName: fmt.Sprint(collectionID),
-		segmentIDLabelName:    fmt.Sprint(segmentID),
-	})
-	DataCoordStoredIndexFilesSize.DeletePartialMatch(prometheus.Labels{
-		databaseLabelName:     dbName,
-		collectionIDLabelName: fmt.Sprint(collectionID),
-		segmentIDLabelName:    fmt.Sprint(segmentID),
-	})
-}
-
 func CleanupDataCoordWithCollectionID(collectionID int64) {
 	IndexTaskNum.DeletePartialMatch(prometheus.Labels{
 		collectionIDLabelName: fmt.Sprint(collectionID),

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -102,7 +102,6 @@ const (
 	indexName                = "index_name"
 	isVectorIndex            = "is_vector_index"
 	segmentStateLabelName    = "segment_state"
-	segmentIDLabelName       = "segment_id"
 	segmentLevelLabelName    = "segment_level"
 	segmentIsSortedLabelName = "segment_is_sorted"
 	usernameLabelName        = "username"