[fix](memory) Fix metadata memory tracking and profile (#44739)

### What problem does this PR solve?

Problem Summary:

1. `MemTrackerLimiter` and `MemoryProfile` add metadata and cache type.
2. metadata memory consists of rowsets, segments, and tablets.
3. `all_segments_mem_tracker` excludes the segments in SegmentCache, and
all segments use the same memory calculation method.
4. segments memory is estimated, which is not accurate and can only be
used as a reference.
5. rowsets and tablets only count part of the memory, mainly PB.

TODO, Metadata also has some memory that can be allocated using Doris
Allocator.

```
MemoryProfile:
  MemoryOverviewSnapshot:
     - PhysicalMemory(VmRSS) Current: 5.85 GB (Peak: 7.13 GB)
     - VirtualMemory(VmSize) Current: 61.35 GB (Peak: 61.67 GB)
    UntrackedMemory:
       - Memory Current: 3.61 GB (Peak: 3.77 GB)
    TrackedMemory:
       - Memory Current: 2.24 GB (Peak: 3.48 GB)
      TasksMemory:
         - Memory Current: 0 (Peak: 749.17 MB)
           - ReservedMemory Current: 0 (Peak: 0)
        Details:
           - Compaction Current: 0 (Peak: 13.60 MB)
           - Load Current: 0 (Peak: 749.17 MB)
             - AllMemTablesMemory Current: 0 (Peak: 853.20 MB)
           - Other Current: 0 (Peak: 0)
           - Query Current: 0 (Peak: 65.20 MB)
           - SchemaChange Current: 0 (Peak: 0)
      GlobalMemory:
         - Memory Current: 55.03 MB (Peak: 70.39 MB)
      MetadataMemory:
         - Memory Current: 972.13 MB (Peak: 972.13 MB)
      CacheMemory:
         - Memory Current: 360.21 MB (Peak: 675.32 MB)
      JemallocMemory:
         - Memory Current: 905.55 MB (Peak: 1.80 GB)
        Details:
           - Cache Current: 646.47 MB (Peak: 1.59 GB)
           - Metadata Current: 259.08 MB (Peak: 259.72 MB)
  GlobalMemorySnapshot:
    Orphan@global@id=3144f940fa84f930-d82d459289789494:
       - Memory Current: 1.73 MB (Peak: 1.73 MB)
    IOBufBlockMemory@global@id=1f4316bc5fb4c3d4-73f14d3652ed269b:
       - Memory Current: 2.65 GB (Peak: 2.65 GB)
    PointQueryExecutor@global@id=794878deaa4d8f56-769fbc87ccb7ec8d:
       - Memory Current: 36.90 MB (Peak: 36.90 MB)
    BlockCompression@global@id=0a4d429b1915dd8b-c2e429b1186d69a8:
       - Memory Current: 563.01 MB (Peak: 563.01 MB)
    RowIdStorageReader@global@id=a546ca5878c9204e-c4c72ea9088ca9a0:
       - Memory Current: 101.25 KB (Peak: 101.25 KB)
    SubcolumnsTree@global@id=294bde93c01d4ba6-4ce7b3290489a1b8:
       - Memory Current: 95.61 MB (Peak: 95.61 MB)
    S3FileBuffer@global@id=bc437d624d9a234d-03eb4bbc3d030a99:
       - Memory Current: 0 (Peak: 0)
  MetadataMemorySnapshot:
    Tablets(not in SchemaCache, TabletSchemaCache)@metadata@id=7646d676ad9f8de9-98beb5f1a8aa739b:
       - Memory Current: 26.84 GB (Peak: 26.84 GB)
    Segments(not in SegmentCache)@metadata@id=864b4887722b327a-516c912d4602b59b:
       - Memory Current: 4.98 MB (Peak: 4.98 MB)
    Rowsets@metadata@id=f0440aa247730497-752a527c71e3c393:
       - Memory Current: 5.57 GB (Peak: 5.57 GB)
    SegmentCache[size]@metadata@id=b34e798db2c25109-04c7ff3c722e4cbb:
       - Memory Current: 4.86 GB (Peak: 4.86 GB)
    SchemaCache[number]@metadata@id=9f41f61531da3fd9-163834a6a6f83eba:
       - Memory Current: 103.38 MB (Peak: 103.38 MB)
    TabletSchemaCache[number]@metadata@id=f9475c16ba7ea1ab-1d28b38568ef0b8d:
       - Memory Current: 170.65 MB (Peak: 170.65 MB)
  CacheMemorySnapshot:
    QueryCache@cache@id=504af426a68103cf-a60f36a48bc1998a:
       - Memory Current: 0 (Peak: 0)
    DataPageCache[size]@cache@id=f744bb4ef53c307b-4c021a8f9b8a77a5:
       - Memory Current: 14.33 GB (Peak: 14.33 GB)
    IndexPageCache[size]@cache@id=cb4ed0e456f081d4-d635edda9aa2778b:
       - Memory Current: 5.25 GB (Peak: 5.25 GB)
    PKIndexPageCache[size]@cache@id=8e4bf12b4d39133a-e0d878a18d73619b:
       - Memory Current: 2.72 GB (Peak: 2.72 GB)
    PointQueryRowCache[size]@cache@id=024dc624c80205f1-ca3221a154e4b68a:
       - Memory Current: 94.84 KB (Peak: 94.84 KB)
    CommonObjLRUCache[number]@cache@id=af40d4a03b391540-43338381c8e826a4:
       - Memory Current: 0 (Peak: 0)
    PointQueryLookupConnectionCache[number]@cache@id=284fa848dd97ccc8-56c61386a4950784:
       - Memory Current: 337.97 KB (Peak: 337.97 KB)
    InvertedIndexSearcherCache[size]@cache@id=f5404416ad0a9465-3cf12f5224276d8a:
       - Memory Current: 111.52 MB (Peak: 111.52 MB)
    InvertedIndexQueryCache[size]@cache@id=6f4d3c55508e3531-3fafe328c6d3e1ab:
       - Memory Current: 51.97 MB (Peak: 51.97 MB)
    QueryCache[size]@cache@id=274d47670e0796ad-542ad958334ff988:
       - Memory Current: 0 (Peak: 0)
    LastSuccessChannelCache[size]@cache@id=08449badfd6abb81-5b30599bfc60f79f:
       - Memory Current: 0 (Peak: 0)
    TabletColumnObjectPool[number]@cache@id=7b42da8391945265-06697f83a6bf01a7:
       - Memory Current: 151.67 MB (Peak: 151.67 MB)
    MowTabletVersionCache[number]@cache@id=f345a1adbd2a87dd-74b9caa3b30cf18b:
       - Memory Current: 150.94 MB (Peak: 150.94 MB)
    CreateTabletRRIdxCache[number]@cache@id=f043eee91e9aef2d-ef3a2245d50545ae:
       - Memory Current: 15.20 MB (Peak: 15.20 MB)
    MowDeleteBitmapAggCache[size]@cache@id=df47a8ed5bf3f11f-ac944d7d7e79d4b3:
       - Memory Current: 181.11 MB (Peak: 181.11 MB)
  TopMemoryTasksSnapshot:
    SnapshotManager@other_task@id=11480ad6017368b9-8aef2150427d229f:
       - Memory Current: 0 (Peak: 0)
    StreamLoadPipe@load@id=024958077445f4f7-d739fd5231dca890:
       - Memory Current: 1.39 GB (Peak: 1.39 GB)
    SegCompaction@compaction@id=2d4d5c796a9f48ee-fdf5dd9f8faf83a8:
       - Memory Current: 0 (Peak: 0)
    Load#Id=544468c7f5a9bac6-2c78fc9962f3908d@load@id=52497774c12c5ad1-1fda35667a3d0e8b:
       - Limit: 2.00 GB
       - Memory Current: 6.06 MB (Peak: 6.06 MB)
    Load#Id=624e7947099ef183-577a22cca9d34d89@load@id=ed4691ce5a748ca8-e49ef7cc9be10b95:
       - Limit: 2.00 GB
       - Memory Current: 6.06 MB (Peak: 6.06 MB)
 ```

be/src/common/daemon.cpp

@@ -500,15 +500,18 @@ void Daemon::cache_adjust_capacity_thread() {
 void Daemon::cache_prune_stale_thread() {
     int32_t interval = config::cache_periodic_prune_stale_sweep_sec;
     while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
-        if (interval <= 0) {
-            LOG(WARNING) << "config of cache clean interval is illegal: [" << interval
-                         << "], force set to 3600 ";
-            interval = 3600;
+        if (config::cache_periodic_prune_stale_sweep_sec <= 0) {
+            LOG(WARNING) << "config of cache clean interval is: [" << interval
+                         << "], it means the cache prune stale thread is disabled, will wait 3s "
+                            "and check again.";
+            interval = 3;
+            continue;
         }
         if (config::disable_memory_gc) {
             continue;
         }
         CacheManager::instance()->for_each_cache_prune_stale();
+        interval = config::cache_periodic_prune_stale_sweep_sec;
     }
 }
 

be/src/olap/delta_writer_v2.h

@@ -46,7 +46,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class Schema;
 class StorageEngine;
 class TupleDescriptor;

be/src/olap/memtable_writer.h

@@ -45,7 +45,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;

be/src/olap/metadata_adder.h

@@ -20,15 +20,17 @@
 #include <bvar/bvar.h>
 #include <stdint.h>
 
+#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
 #include "util/runtime_profile.h"
 
 namespace doris {
 
 inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
 
-inline bvar::Adder<int64_t> g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
-inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");
+inline bvar::Adder<int64_t> g_rowset_mem_bytes("doris_rowset_mem_bytes");
+inline bvar::Adder<int64_t> g_rowset_num("doris_rowset_num");
 
 inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
@@ -42,8 +44,9 @@ inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
 inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
 
-inline bvar::Adder<int64_t> g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
-inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");
+inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
+inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+inline bvar::Adder<int64_t> g_segment_estimate_mem_bytes("doris_segment_estimate_mem_bytes");
 
 inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
 inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
@@ -96,6 +99,10 @@ class ZoneMapIndexReader;
     When a derived Class extends MetadataAdder, then the Class's number and fixed length field's memory can be counted automatically.
     But if the Class has variable length field, then you should overwrite get_metadata_size and call update_metadata_size when the Class's memory changes.
 
+    get_metadata_size is only the memory of the metadata object itself, not include child objects,
+    for example, TabletMeta::get_metadata_size does not include the memory of TabletSchema.
+    Note, the memory allocated by Doris Allocator is not included.
+
     There are some special situations that need to be noted:
     1. when the derived Class override copy constructor, you'd better update memory size(call update_metadata_size) if derived class's 
     memory changed in its copy constructor or you not call MetadataAdder's copy constructor.
@@ -111,6 +118,31 @@ class MetadataAdder {
 
     static void dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot);
 
+    static int64_t get_all_tablets_size() {
+        return g_tablet_meta_mem_bytes.get_value() + g_tablet_column_mem_bytes.get_value() +
+               g_tablet_index_mem_bytes.get_value() + g_tablet_schema_mem_bytes.get_value();
+    }
+
+    static int64_t get_all_rowsets_size() {
+        return g_rowset_meta_mem_bytes.get_value() + g_rowset_mem_bytes.get_value();
+    }
+
+    static int64_t get_all_segments_size() {
+        return g_segment_mem_bytes.get_value() + g_column_reader_mem_bytes.get_value() +
+               g_bitmap_index_reader_mem_bytes.get_value() +
+               g_bloom_filter_index_reader_mem_bytes.get_value() +
+               g_index_page_reader_mem_bytes.get_value() +
+               g_indexed_column_reader_mem_bytes.get_value() +
+               g_inverted_index_reader_mem_bytes.get_value() +
+               g_ordinal_index_reader_mem_bytes.get_value() +
+               g_zone_map_index_reader_mem_bytes.get_value();
+    }
+
+    // Doris currently uses the estimated segments memory as the basis, maybe it is more realistic.
+    static int64_t get_all_segments_estimate_size() {
+        return g_segment_estimate_mem_bytes.get_value();
+    }
+
 protected:
     MetadataAdder(const MetadataAdder& other);
 
@@ -122,7 +154,6 @@ class MetadataAdder {
 
     MetadataAdder<T>& operator=(const MetadataAdder<T>& other) = default;
 
-private:
     int64_t _current_meta_size {0};
 
     void add_mem_size(int64_t val);
@@ -167,7 +198,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_mem_bytes << val;
+        g_rowset_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -177,7 +208,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_mem_bytes << val;
+        g_segment_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -208,7 +239,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_num << val;
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_num << val;
+        g_rowset_num << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_num << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -218,7 +249,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_num << val;
+        g_segment_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -250,12 +281,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value());
     COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());
 
-    RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_rowsets_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT);
-    COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value());
-    COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());
+    RuntimeProfile::Counter* rowset_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* rowset_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetNum", TUnit::UNIT);
+    COUNTER_SET(rowset_mem_bytes_counter, g_rowset_mem_bytes.get_value());
+    COUNTER_SET(rowset_num_counter, g_rowset_num.get_value());
 
     RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES);
@@ -285,12 +316,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value());
     COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());
 
-    RuntimeProfile::Counter* all_segments_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_segments_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT);
-    COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value());
-    COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());
+    RuntimeProfile::Counter* segment_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* segment_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentNum", TUnit::UNIT);
+    COUNTER_SET(segment_mem_bytes_counter, g_segment_mem_bytes.get_value());
+    COUNTER_SET(segment_num_counter, g_segment_num.get_value());
 
     RuntimeProfile::Counter* column_reader_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES);

be/src/olap/rowset/segment_v2/indexed_column_reader.cpp

@@ -81,7 +81,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.ordinal_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.ordinal_index_meta().root_page(),
-                                            &_ordinal_index_page_handle, &_ordinal_index_reader));
+                                            &_ordinal_index_page_handle,
+                                            _ordinal_index_reader.get()));
             _has_index_page = true;
         }
     }
@@ -92,7 +93,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.value_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.value_index_meta().root_page(),
-                                            &_value_index_page_handle, &_value_index_reader));
+                                            &_value_index_page_handle, _value_index_reader.get()));
             _has_index_page = true;
         }
     }

be/src/olap/rowset/segment_v2/indexed_column_reader.h

@@ -50,9 +50,12 @@ class EncodingInfo;
 class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
 public:
     explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta)
-            : _file_reader(std::move(file_reader)), _meta(meta) {}
+            : _file_reader(std::move(file_reader)), _meta(meta) {
+        _ordinal_index_reader = std::make_unique<IndexPageReader>();
+        _value_index_reader = std::make_unique<IndexPageReader>();
+    }
 
-    ~IndexedColumnReader();
+    ~IndexedColumnReader() override;
 
     Status load(bool use_page_cache, bool kept_in_memory,
                 OlapReaderStatistics* index_load_stats = nullptr);
@@ -90,8 +93,8 @@ class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
     bool _has_index_page = false;
     // valid only when the column contains only one data page
     PagePointer _sole_data_page;
-    IndexPageReader _ordinal_index_reader;
-    IndexPageReader _value_index_reader;
+    std::unique_ptr<IndexPageReader> _ordinal_index_reader;
+    std::unique_ptr<IndexPageReader> _value_index_reader;
     PageHandle _ordinal_index_page_handle;
     PageHandle _value_index_page_handle;
 
@@ -108,8 +111,8 @@ class IndexedColumnIterator {
     explicit IndexedColumnIterator(const IndexedColumnReader* reader,
                                    OlapReaderStatistics* stats = nullptr)
             : _reader(reader),
-              _ordinal_iter(&reader->_ordinal_index_reader),
-              _value_iter(&reader->_value_index_reader),
+              _ordinal_iter(reader->_ordinal_index_reader.get()),
+              _value_iter(reader->_value_index_reader.get()),
               _stats(stats) {}
 
     // Seek to the given ordinal entry. Entry 0 is the first entry.

be/src/olap/rowset/segment_v2/page_handle.h

@@ -23,6 +23,10 @@
 #include "util/slice.h" // for Slice
 
 namespace doris {
+
+// After disable page cache, sometimes we need to know the percentage of data pages in query memory.
+inline bvar::Adder<int64_t> g_page_no_cache_mem_bytes("doris_page_no_cache_mem_bytes");
+
 namespace segment_v2 {
 
 // When a column page is read into memory, we use this to store it.
@@ -37,8 +41,7 @@ class PageHandle {
     // This class will take the ownership of input data's memory. It will
     // free it when deconstructs.
     PageHandle(DataPage* data) : _is_data_owner(true), _data(data) {
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
-        _page_tracker->consume(_data->capacity());
+        g_page_no_cache_mem_bytes << _data->capacity();
     }
 
     // This class will take the content of cache data, and will make input
@@ -51,20 +54,18 @@ class PageHandle {
         // we can use std::exchange if we switch c++14 on
         std::swap(_is_data_owner, other._is_data_owner);
         std::swap(_data, other._data);
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
     }
 
     PageHandle& operator=(PageHandle&& other) noexcept {
         std::swap(_is_data_owner, other._is_data_owner);
         std::swap(_data, other._data);
         _cache_data = std::move(other._cache_data);
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
         return *this;
     }
 
     ~PageHandle() {
         if (_is_data_owner) {
-            _page_tracker->release(_data->capacity());
+            g_page_no_cache_mem_bytes << -_data->capacity();
             delete _data;
         } else {
             DCHECK(_data == nullptr);
@@ -85,7 +86,6 @@ class PageHandle {
     // otherwise _cache_data is valid, and data is belong to cache.
     bool _is_data_owner = false;
     DataPage* _data = nullptr;
-    std::shared_ptr<MemTracker> _page_tracker;
     PageCacheHandle _cache_data;
 
     // Don't allow copy and assign

be/src/olap/rowset/segment_v2/segment.cpp

@@ -163,7 +163,11 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr table
           _tablet_schema(std::move(tablet_schema)),
           _idx_file_info(idx_file_info) {}
 
-Segment::~Segment() = default;
+Segment::~Segment() {
+    g_segment_estimate_mem_bytes << -_tracked_meta_mem_usage;
+    // if failed, fix `_tracked_meta_mem_usage` accuracy
+    DCHECK(_tracked_meta_mem_usage == meta_mem_usage());
+}
 
 io::UInt128Wrapper Segment::file_cache_key(std::string_view rowset_id, uint32_t seg_id) {
     return io::BlockFileCache::hash(fmt::format("{}_{}.dat", rowset_id, seg_id));
@@ -174,6 +178,12 @@ int64_t Segment::get_metadata_size() const {
            (_pk_index_meta ? _pk_index_meta->ByteSizeLong() : 0);
 }
 
+void Segment::update_metadata_size() {
+    MetadataAdder::update_metadata_size();
+    g_segment_estimate_mem_bytes << _meta_mem_usage - _tracked_meta_mem_usage;
+    _tracked_meta_mem_usage = _meta_mem_usage;
+}
+
 Status Segment::_open() {
     _footer_pb = std::make_unique<SegmentFooterPB>();
     RETURN_IF_ERROR(_parse_footer(_footer_pb.get()));
@@ -191,8 +201,6 @@ Status Segment::_open() {
         _meta_mem_usage += _pk_index_meta->ByteSizeLong();
     }
 
-    update_metadata_size();
-
     _meta_mem_usage += sizeof(*this);
     _meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
 
@@ -201,6 +209,8 @@ Status Segment::_open() {
     // 0.01 comes from PrimaryKeyIndexBuilder::init
     _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
 
+    update_metadata_size();
+
     return Status::OK();
 }
 
@@ -467,6 +477,7 @@ Status Segment::_load_pk_bloom_filter() {
         // for BE UT "segment_cache_test"
         return _load_pk_bf_once.call([this] {
             _meta_mem_usage += 100;
+            update_metadata_size();
             return Status::OK();
         });
     }

be/src/olap/rowset/segment_v2/segment.h

@@ -57,7 +57,6 @@ class IDataType;
 class ShortKeyIndexDecoder;
 class Schema;
 class StorageReadOptions;
-class MemTracker;
 class PrimaryKeyIndexReader;
 class RowwiseIterator;
 struct RowLocation;
@@ -93,6 +92,7 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
     ~Segment();
 
     int64_t get_metadata_size() const override;
+    void update_metadata_size();
 
     Status new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options,
                         std::unique_ptr<RowwiseIterator>* iter);
@@ -163,6 +163,8 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
 
     io::FileReaderSPtr file_reader() { return _file_reader; }
 
+    // Including the column reader memory.
+    // another method `get_metadata_size` not include the column reader, only the segment object itself.
     int64_t meta_mem_usage() const { return _meta_mem_usage; }
 
     // Identify the column by unique id or path info
@@ -249,9 +251,8 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
     // 1. Tracking memory use by segment meta data such as footer or index page.
     // 2. Tracking memory use by segment column reader
     // The memory consumed by querying is tracked in segment iterator.
-    // TODO: Segment::_meta_mem_usage Unknown value overflow, causes the value of SegmentMeta mem tracker
-    // is similar to `-2912341218700198079`. So, temporarily put it in experimental type tracker.
     int64_t _meta_mem_usage;
+    int64_t _tracked_meta_mem_usage = 0;
 
     RowsetId _rowset_id;
     TabletSchemaSPtr _tablet_schema;

be/src/olap/rowset_builder.h

@@ -38,7 +38,6 @@ namespace doris {
 class CalcDeleteBitmapToken;
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;