1

be/src/common/daemon.cpp

@@ -500,15 +500,17 @@ void Daemon::cache_adjust_capacity_thread() {
 void Daemon::cache_prune_stale_thread() {
     int32_t interval = config::cache_periodic_prune_stale_sweep_sec;
     while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
-        if (interval <= 0) {
-            LOG(WARNING) << "config of cache clean interval is illegal: [" << interval
-                         << "], force set to 3600 ";
+        if (config::cache_periodic_prune_stale_sweep_sec <= 0) {
+            LOG(WARNING) << "config of cache clean interval is: [" << interval
+                         << "], cache prune stale thread pause 3600s ";
             interval = 3600;
+            continue;
         }
         if (config::disable_memory_gc) {
             continue;
         }
         CacheManager::instance()->for_each_cache_prune_stale();
+        interval = config::cache_periodic_prune_stale_sweep_sec;
     }
 }
 

be/src/olap/delta_writer_v2.h

@@ -46,7 +46,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class Schema;
 class StorageEngine;
 class TupleDescriptor;

be/src/olap/memtable_writer.h

@@ -45,7 +45,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;

be/src/olap/metadata_adder.h

@@ -20,15 +20,17 @@
 #include <bvar/bvar.h>
 #include <stdint.h>
 
+#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
 #include "util/runtime_profile.h"
 
 namespace doris {
 
 inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
 
-inline bvar::Adder<int64_t> g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
-inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");
+inline bvar::Adder<int64_t> g_rowset_mem_bytes("doris_rowset_mem_bytes");
+inline bvar::Adder<int64_t> g_rowset_num("doris_rowset_num");
 
 inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
@@ -42,8 +44,8 @@ inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
 inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
 
-inline bvar::Adder<int64_t> g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
-inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");
+inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
+inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
 
 inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
 inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
@@ -96,6 +98,10 @@ class ZoneMapIndexReader;
     When a derived Class extends MetadataAdder, then the Class's number and fixed length field's memory can be counted automatically.
     But if the Class has variable length field, then you should overwrite get_metadata_size and call update_metadata_size when the Class's memory changes.
 
+    get_metadata_size is only the memory of the metadata object itself, not include child objects,
+    for example, TabletMeta::get_metadata_size does not include the memory of TabletSchema.
+    Note, the memory allocated by Doris Allocator is not included.
+
     There are some special situations that need to be noted:
     1. when the derived Class override copy constructor, you'd better update memory size(call update_metadata_size) if derived class's 
     memory changed in its copy constructor or you not call MetadataAdder's copy constructor.
@@ -167,7 +173,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_mem_bytes << val;
+        g_rowset_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -177,7 +183,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_mem_bytes << val;
+        g_segment_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -207,18 +213,24 @@ void MetadataAdder<T>::add_num(int64_t val) {
     }
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_num << val;
+        ExecEnv::GetInstance()->all_rowsets_mem_tracker()->consume(val);
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_num << val;
+        g_rowset_num << val;
+        ExecEnv::GetInstance()->all_rowsets_mem_tracker()->consume(val);
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_num << val;
+        ExecEnv::GetInstance()->all_tablets_mem_tracker()->consume(val);
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
         g_tablet_column_num << val;
+        ExecEnv::GetInstance()->all_tablets_mem_tracker()->consume(val);
     } else if constexpr (std::is_same_v<T, TabletIndex>) {
         g_tablet_index_num << val;
+        ExecEnv::GetInstance()->all_tablets_mem_tracker()->consume(val);
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_num << val;
+        ExecEnv::GetInstance()->all_tablets_mem_tracker()->consume(val);
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_num << val;
+        g_segment_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -250,12 +262,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value());
     COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());
 
-    RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_rowsets_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT);
-    COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value());
-    COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());
+    RuntimeProfile::Counter* rowset_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* rowset_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetNum", TUnit::UNIT);
+    COUNTER_SET(rowset_mem_bytes_counter, g_rowset_mem_bytes.get_value());
+    COUNTER_SET(rowset_num_counter, g_rowset_num.get_value());
 
     RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES);
@@ -285,12 +297,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value());
     COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());
 
-    RuntimeProfile::Counter* all_segments_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_segments_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT);
-    COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value());
-    COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());
+    RuntimeProfile::Counter* segment_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* segment_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentNum", TUnit::UNIT);
+    COUNTER_SET(segment_mem_bytes_counter, g_segment_mem_bytes.get_value());
+    COUNTER_SET(segment_num_counter, g_segment_num.get_value());
 
     RuntimeProfile::Counter* column_reader_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES);

be/src/olap/rowset/segment_v2/indexed_column_reader.cpp

@@ -81,7 +81,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.ordinal_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.ordinal_index_meta().root_page(),
-                                            &_ordinal_index_page_handle, &_ordinal_index_reader));
+                                            &_ordinal_index_page_handle,
+                                            _ordinal_index_reader.get()));
             _has_index_page = true;
         }
     }
@@ -92,7 +93,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.value_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.value_index_meta().root_page(),
-                                            &_value_index_page_handle, &_value_index_reader));
+                                            &_value_index_page_handle, _value_index_reader.get()));
             _has_index_page = true;
         }
     }

be/src/olap/rowset/segment_v2/indexed_column_reader.h

@@ -50,9 +50,12 @@ class EncodingInfo;
 class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
 public:
     explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta)
-            : _file_reader(std::move(file_reader)), _meta(meta) {}
+            : _file_reader(std::move(file_reader)), _meta(meta) {
+        _ordinal_index_reader = std::make_unique<IndexPageReader>();
+        _value_index_reader = std::make_unique<IndexPageReader>();
+    }
 
-    ~IndexedColumnReader();
+    ~IndexedColumnReader() override;
 
     Status load(bool use_page_cache, bool kept_in_memory,
                 OlapReaderStatistics* index_load_stats = nullptr);
@@ -90,8 +93,8 @@ class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
     bool _has_index_page = false;
     // valid only when the column contains only one data page
     PagePointer _sole_data_page;
-    IndexPageReader _ordinal_index_reader;
-    IndexPageReader _value_index_reader;
+    std::unique_ptr<IndexPageReader> _ordinal_index_reader;
+    std::unique_ptr<IndexPageReader> _value_index_reader;
     PageHandle _ordinal_index_page_handle;
     PageHandle _value_index_page_handle;
 
@@ -108,8 +111,8 @@ class IndexedColumnIterator {
     explicit IndexedColumnIterator(const IndexedColumnReader* reader,
                                    OlapReaderStatistics* stats = nullptr)
             : _reader(reader),
-              _ordinal_iter(&reader->_ordinal_index_reader),
-              _value_iter(&reader->_value_index_reader),
+              _ordinal_iter(reader->_ordinal_index_reader.get()),
+              _value_iter(reader->_value_index_reader.get()),
               _stats(stats) {}
 
     // Seek to the given ordinal entry. Entry 0 is the first entry.

be/src/olap/rowset/segment_v2/segment.cpp

@@ -163,7 +163,10 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr table
           _tablet_schema(std::move(tablet_schema)),
           _idx_file_info(idx_file_info) {}
 
-Segment::~Segment() = default;
+Segment::~Segment() {
+    ExecEnv::GetInstance()->all_segments_mem_tracker()->release(_tracked_meta_mem_usage);
+    DCHECK(_tracked_meta_mem_usage == meta_mem_usage());
+}
 
 io::UInt128Wrapper Segment::file_cache_key(std::string_view rowset_id, uint32_t seg_id) {
     return io::BlockFileCache::hash(fmt::format("{}_{}.dat", rowset_id, seg_id));
@@ -200,6 +203,8 @@ Status Segment::_open() {
     _meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
     // 0.01 comes from PrimaryKeyIndexBuilder::init
     _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
+    _tracked_meta_mem_usage = _meta_mem_usage;
+    ExecEnv::GetInstance()->all_segments_mem_tracker()->consume(_tracked_meta_mem_usage);
 
     return Status::OK();
 }

be/src/olap/rowset/segment_v2/segment.h

@@ -57,7 +57,6 @@ class IDataType;
 class ShortKeyIndexDecoder;
 class Schema;
 class StorageReadOptions;
-class MemTracker;
 class PrimaryKeyIndexReader;
 class RowwiseIterator;
 struct RowLocation;
@@ -163,6 +162,8 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
 
     io::FileReaderSPtr file_reader() { return _file_reader; }
 
+    // Including the column reader memory.
+    // another method `get_metadata_size` not include the column reader, only the segment object itself.
     int64_t meta_mem_usage() const { return _meta_mem_usage; }
 
     // Identify the column by unique id or path info
@@ -252,6 +253,7 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
     // TODO: Segment::_meta_mem_usage Unknown value overflow, causes the value of SegmentMeta mem tracker
     // is similar to `-2912341218700198079`. So, temporarily put it in experimental type tracker.
     int64_t _meta_mem_usage;
+    int64_t _tracked_meta_mem_usage;
 
     RowsetId _rowset_id;
     TabletSchemaSPtr _tablet_schema;

be/src/olap/rowset_builder.h

@@ -38,7 +38,6 @@ namespace doris {
 class CalcDeleteBitmapToken;
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;

be/src/olap/schema_cache.h

@@ -76,7 +76,8 @@ class SchemaCache : public LRUCachePolicy {
         auto* value = new CacheValue;
         value->tablet_schema = schema;
 
-        auto* lru_handle = insert(key, value, 1, schema->mem_size(), CachePriority::NORMAL);
+        auto* lru_handle =
+                insert(key, value, 1, schema->get_metadata_size(), CachePriority::NORMAL);
         release(lru_handle);
     }
 

be/src/olap/segment_loader.cpp

@@ -77,9 +77,8 @@ Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
         }
         if (use_cache && !config::disable_segment_cache) {
             // memory of SegmentCache::CacheValue will be handled by SegmentCache
-            auto* cache_value = new SegmentCache::CacheValue();
+            auto* cache_value = new SegmentCache::CacheValue(segment);
             _cache_mem_usage += segment->meta_mem_usage();
-            cache_value->segment = std::move(segment);
             _segment_cache->insert(cache_key, *cache_value, cache_handle);
         } else {
             cache_handle->push_segment(std::move(segment));

be/src/olap/segment_loader.h

@@ -75,9 +75,15 @@ class SegmentCache : public LRUCachePolicy {
     // Holding all opened segments of a rowset.
     class CacheValue : public LRUCacheValueBase {
     public:
-        ~CacheValue() override { segment.reset(); }
+        CacheValue(segment_v2::SegmentSharedPtr segment_) : segment(std::move(segment_)) {
+            // Transfer to segment cache memory tracker.
+            ExecEnv::GetInstance()->all_segments_mem_tracker()->release(segment->meta_mem_usage());
+        }
+        ~CacheValue() override {
+            ExecEnv::GetInstance()->all_segments_mem_tracker()->consume(segment->meta_mem_usage());
+        }
 
-        segment_v2::SegmentSharedPtr segment;
+        const segment_v2::SegmentSharedPtr segment;
     };
 
     SegmentCache(size_t memory_bytes_limit, size_t segment_num_limit)

be/src/olap/tablet_manager.cpp

@@ -57,8 +57,6 @@
 #include "olap/tablet_schema.h"
 #include "olap/txn_manager.h"
 #include "runtime/exec_env.h"
-#include "runtime/memory/mem_tracker.h"
-#include "runtime/thread_context.h"
 #include "service/backend_options.h"
 #include "util/defer_op.h"
 #include "util/doris_metrics.h"
@@ -83,28 +81,18 @@ using std::vector;
 namespace doris {
 using namespace ErrorCode;
 
-DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_meta_mem_consumption, MetricUnit::BYTES, "",
-                                   mem_consumption, Labels({{"type", "tablet_meta"}}));
-
 bvar::Adder<int64_t> g_tablet_meta_schema_columns_count("tablet_meta_schema_columns_count");
 
 TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size)
         : _engine(engine),
-          _tablet_meta_mem_tracker(std::make_shared<MemTracker>("TabletMeta(experimental)")),
           _tablets_shards_size(tablet_map_lock_shard_size),
           _tablets_shards_mask(tablet_map_lock_shard_size - 1) {
     CHECK_GT(_tablets_shards_size, 0);
     CHECK_EQ(_tablets_shards_size & _tablets_shards_mask, 0);
     _tablets_shards.resize(_tablets_shards_size);
-    REGISTER_HOOK_METRIC(tablet_meta_mem_consumption,
-                         [this]() { return _tablet_meta_mem_tracker->consumption(); });
 }
 
-TabletManager::~TabletManager() {
-#ifndef BE_TEST
-    DEREGISTER_HOOK_METRIC(tablet_meta_mem_consumption);
-#endif
-}
+TabletManager::~TabletManager() = default;
 
 Status TabletManager::_add_tablet_unlocked(TTabletId tablet_id, const TabletSharedPtr& tablet,
                                            bool update_meta, bool force, RuntimeProfile* profile) {
@@ -242,10 +230,6 @@ Status TabletManager::_add_tablet_to_map_unlocked(TTabletId tablet_id,
     tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
     tablet_map[tablet_id] = tablet;
     _add_tablet_to_partition(tablet);
-    // TODO: remove multiply 2 of tablet meta mem size
-    // Because table schema will copy in tablet, there will be double mem cost
-    // so here multiply 2
-    _tablet_meta_mem_tracker->consume(tablet->tablet_meta()->mem_size() * 2);
     g_tablet_meta_schema_columns_count << tablet->tablet_meta()->tablet_columns_num();
     COUNTER_UPDATE(ADD_CHILD_TIMER(profile, "RegisterTabletInfo", "AddTablet"),
                    static_cast<int64_t>(watch.reset()));
@@ -599,7 +583,6 @@ Status TabletManager::_drop_tablet(TTabletId tablet_id, TReplicaId replica_id, b
     }
 
     to_drop_tablet->deregister_tablet_from_dir();
-    _tablet_meta_mem_tracker->release(to_drop_tablet->tablet_meta()->mem_size() * 2);
     g_tablet_meta_schema_columns_count << -to_drop_tablet->tablet_meta()->tablet_columns_num();
     return Status::OK();
 }

be/src/olap/tablet_manager.h

@@ -251,9 +251,6 @@ class TabletManager {
 
     StorageEngine& _engine;
 
-    // TODO: memory size of TabletSchema cannot be accurately tracked.
-    std::shared_ptr<MemTracker> _tablet_meta_mem_tracker;
-
     const int32_t _tablets_shards_size;
     const int32_t _tablets_shards_mask;
     std::vector<tablets_shard> _tablets_shards;

be/src/olap/tablet_meta.cpp

@@ -781,12 +781,6 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
             time_series_compaction_level_threshold());
 }
 
-int64_t TabletMeta::mem_size() const {
-    auto size = sizeof(TabletMeta);
-    size += _schema->mem_size();
-    return size;
-}
-
 void TabletMeta::to_json(string* json_string, json2pb::Pb2JsonOptions& options) {
     TabletMetaPB tablet_meta_pb;
     to_meta_pb(&tablet_meta_pb);