diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index ce2a6878dba034..eabdc047260902 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -500,15 +500,17 @@ void Daemon::cache_adjust_capacity_thread() {
 void Daemon::cache_prune_stale_thread() {
     int32_t interval = config::cache_periodic_prune_stale_sweep_sec;
     while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
-        if (interval <= 0) {
-            LOG(WARNING) << "config of cache clean interval is illegal: [" << interval
-                         << "], force set to 3600 ";
+        if (config::cache_periodic_prune_stale_sweep_sec <= 0) {
+            LOG(WARNING) << "config of cache clean interval is: [" << interval
+                         << "], cache prune stale thread pause 3600s ";
             interval = 3600;
+            continue;
         }
         if (config::disable_memory_gc) {
             continue;
         }
         CacheManager::instance()->for_each_cache_prune_stale();
+        interval = config::cache_periodic_prune_stale_sweep_sec;
     }
 }
 
diff --git a/be/src/olap/delta_writer_v2.h b/be/src/olap/delta_writer_v2.h
index beeb3d3ecd3ec5..f9c2800a68f499 100644
--- a/be/src/olap/delta_writer_v2.h
+++ b/be/src/olap/delta_writer_v2.h
@@ -46,7 +46,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class Schema;
 class StorageEngine;
 class TupleDescriptor;
diff --git a/be/src/olap/memtable_writer.h b/be/src/olap/memtable_writer.h
index fb07e740fa3cf6..713400793a1754 100644
--- a/be/src/olap/memtable_writer.h
+++ b/be/src/olap/memtable_writer.h
@@ -45,7 +45,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;
diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
index 559c5db873b2f6..df329158c473ab 100644
--- a/be/src/olap/metadata_adder.h
+++ b/be/src/olap/metadata_adder.h
@@ -20,6 +20,8 @@
 #include <bvar/bvar.h>
 #include <stdint.h>
 
+#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
 #include "util/runtime_profile.h"
 
 namespace doris {
@@ -27,8 +29,8 @@ namespace doris {
 inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
 
-inline bvar::Adder<int64_t> g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
-inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");
+inline bvar::Adder<int64_t> g_rowset_mem_bytes("doris_rowset_mem_bytes");
+inline bvar::Adder<int64_t> g_rowset_num("doris_rowset_num");
 
 inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
@@ -42,8 +44,9 @@ inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
 inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
 
-inline bvar::Adder<int64_t> g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
-inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");
+inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
+inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+inline bvar::Adder<int64_t> g_segment_estimate_mem_bytes("doris_segment_estimate_mem_bytes");
 
 inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
 inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
@@ -96,6 +99,10 @@ class ZoneMapIndexReader;
     When a derived Class extends MetadataAdder, then the Class's number and fixed length field's memory can be counted automatically.
     But if the Class has variable length field, then you should overwrite get_metadata_size and call update_metadata_size when the Class's memory changes.
 
+    get_metadata_size is only the memory of the metadata object itself, not include child objects,
+    for example, TabletMeta::get_metadata_size does not include the memory of TabletSchema.
+    Note, the memory allocated by Doris Allocator is not included.
+
     There are some special situations that need to be noted:
     1. when the derived Class override copy constructor, you'd better update memory size(call update_metadata_size) if derived class's 
     memory changed in its copy constructor or you not call MetadataAdder's copy constructor.
@@ -111,6 +118,31 @@ class MetadataAdder {
 
     static void dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot);
 
+    static int64_t get_all_tablets_size() {
+        return g_tablet_meta_mem_bytes.get_value() + g_tablet_column_mem_bytes.get_value() +
+               g_tablet_index_mem_bytes.get_value() + g_tablet_schema_mem_bytes.get_value();
+    }
+
+    static int64_t get_all_rowsets_size() {
+        return g_rowset_meta_mem_bytes.get_value() + g_rowset_mem_bytes.get_value();
+    }
+
+    static int64_t get_all_segments_size() {
+        return g_segment_mem_bytes.get_value() + g_column_reader_mem_bytes.get_value() +
+               g_bitmap_index_reader_mem_bytes.get_value() +
+               g_bloom_filter_index_reader_mem_bytes.get_value() +
+               g_index_page_reader_mem_bytes.get_value() +
+               g_indexed_column_reader_mem_bytes.get_value() +
+               g_inverted_index_reader_mem_bytes.get_value() +
+               g_ordinal_index_reader_mem_bytes.get_value() +
+               g_zone_map_index_reader_mem_bytes.get_value();
+    }
+
+    // Doris currently uses the estimated segments memory as the basis, maybe it is more realistic.
+    static int64_t get_all_segments_estimate_size() {
+        return g_segment_estimate_mem_bytes.get_value();
+    }
+
 protected:
     MetadataAdder(const MetadataAdder& other);
 
@@ -167,7 +199,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_mem_bytes << val;
+        g_rowset_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -177,7 +209,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_mem_bytes << val;
+        g_segment_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -208,7 +240,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_num << val;
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_num << val;
+        g_rowset_num << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_num << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -218,7 +250,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_num << val;
+        g_segment_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -250,12 +282,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value());
     COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());
 
-    RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_rowsets_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT);
-    COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value());
-    COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());
+    RuntimeProfile::Counter* rowset_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* rowset_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetNum", TUnit::UNIT);
+    COUNTER_SET(rowset_mem_bytes_counter, g_rowset_mem_bytes.get_value());
+    COUNTER_SET(rowset_num_counter, g_rowset_num.get_value());
 
     RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES);
@@ -285,12 +317,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value());
     COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());
 
-    RuntimeProfile::Counter* all_segments_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_segments_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT);
-    COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value());
-    COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());
+    RuntimeProfile::Counter* segment_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* segment_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentNum", TUnit::UNIT);
+    COUNTER_SET(segment_mem_bytes_counter, g_segment_mem_bytes.get_value());
+    COUNTER_SET(segment_num_counter, g_segment_num.get_value());
 
     RuntimeProfile::Counter* column_reader_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES);
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 3028211f266157..da6beff5d8d6a2 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -81,7 +81,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.ordinal_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.ordinal_index_meta().root_page(),
-                                            &_ordinal_index_page_handle, &_ordinal_index_reader));
+                                            &_ordinal_index_page_handle,
+                                            _ordinal_index_reader.get()));
             _has_index_page = true;
         }
     }
@@ -92,7 +93,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.value_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.value_index_meta().root_page(),
-                                            &_value_index_page_handle, &_value_index_reader));
+                                            &_value_index_page_handle, _value_index_reader.get()));
             _has_index_page = true;
         }
     }
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index c3469f9f6bed0d..c9640c0007c153 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -50,9 +50,12 @@ class EncodingInfo;
 class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
 public:
     explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta)
-            : _file_reader(std::move(file_reader)), _meta(meta) {}
+            : _file_reader(std::move(file_reader)), _meta(meta) {
+        _ordinal_index_reader = std::make_unique<IndexPageReader>();
+        _value_index_reader = std::make_unique<IndexPageReader>();
+    }
 
-    ~IndexedColumnReader();
+    ~IndexedColumnReader() override;
 
     Status load(bool use_page_cache, bool kept_in_memory,
                 OlapReaderStatistics* index_load_stats = nullptr);
@@ -90,8 +93,8 @@ class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
     bool _has_index_page = false;
     // valid only when the column contains only one data page
     PagePointer _sole_data_page;
-    IndexPageReader _ordinal_index_reader;
-    IndexPageReader _value_index_reader;
+    std::unique_ptr<IndexPageReader> _ordinal_index_reader;
+    std::unique_ptr<IndexPageReader> _value_index_reader;
     PageHandle _ordinal_index_page_handle;
     PageHandle _value_index_page_handle;
 
@@ -108,8 +111,8 @@ class IndexedColumnIterator {
     explicit IndexedColumnIterator(const IndexedColumnReader* reader,
                                    OlapReaderStatistics* stats = nullptr)
             : _reader(reader),
-              _ordinal_iter(&reader->_ordinal_index_reader),
-              _value_iter(&reader->_value_index_reader),
+              _ordinal_iter(reader->_ordinal_index_reader.get()),
+              _value_iter(reader->_value_index_reader.get()),
               _stats(stats) {}
 
     // Seek to the given ordinal entry. Entry 0 is the first entry.
diff --git a/be/src/olap/rowset/segment_v2/page_handle.h b/be/src/olap/rowset/segment_v2/page_handle.h
index b1e53ee808697e..d4dfdfb2ff3c55 100644
--- a/be/src/olap/rowset/segment_v2/page_handle.h
+++ b/be/src/olap/rowset/segment_v2/page_handle.h
@@ -23,6 +23,10 @@
 #include "util/slice.h" // for Slice
 
 namespace doris {
+
+// After disable page cache, sometimes we need to know the percentage of data pages in query memory.
+inline bvar::Adder<int64_t> g_page_no_cache_mem_bytes("doris_page_no_cache_mem_bytes");
+
 namespace segment_v2 {
 
 // When a column page is read into memory, we use this to store it.
@@ -37,8 +41,7 @@ class PageHandle {
     // This class will take the ownership of input data's memory. It will
     // free it when deconstructs.
     PageHandle(DataPage* data) : _is_data_owner(true), _data(data) {
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
-        _page_tracker->consume(_data->capacity());
+        g_page_no_cache_mem_bytes << _data->capacity();
     }
 
     // This class will take the content of cache data, and will make input
@@ -51,20 +54,18 @@ class PageHandle {
         // we can use std::exchange if we switch c++14 on
         std::swap(_is_data_owner, other._is_data_owner);
         std::swap(_data, other._data);
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
     }
 
     PageHandle& operator=(PageHandle&& other) noexcept {
         std::swap(_is_data_owner, other._is_data_owner);
         std::swap(_data, other._data);
         _cache_data = std::move(other._cache_data);
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
         return *this;
     }
 
     ~PageHandle() {
         if (_is_data_owner) {
-            _page_tracker->release(_data->capacity());
+            g_page_no_cache_mem_bytes << -_data->capacity();
             delete _data;
         } else {
             DCHECK(_data == nullptr);
@@ -85,7 +86,6 @@ class PageHandle {
     // otherwise _cache_data is valid, and data is belong to cache.
     bool _is_data_owner = false;
     DataPage* _data = nullptr;
-    std::shared_ptr<MemTracker> _page_tracker;
     PageCacheHandle _cache_data;
 
     // Don't allow copy and assign
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 0ad799683fc458..5800346ff18a3d 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -163,7 +163,11 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr table
           _tablet_schema(std::move(tablet_schema)),
           _idx_file_info(idx_file_info) {}
 
-Segment::~Segment() = default;
+Segment::~Segment() {
+    g_segment_estimate_mem_bytes << -_tracked_meta_mem_usage;
+    // if failed, fix `_tracked_meta_mem_usage` accuracy
+    DCHECK(_tracked_meta_mem_usage == meta_mem_usage());
+}
 
 io::UInt128Wrapper Segment::file_cache_key(std::string_view rowset_id, uint32_t seg_id) {
     return io::BlockFileCache::hash(fmt::format("{}_{}.dat", rowset_id, seg_id));
@@ -200,6 +204,8 @@ Status Segment::_open() {
     _meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
     // 0.01 comes from PrimaryKeyIndexBuilder::init
     _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
+    g_segment_estimate_mem_bytes << _meta_mem_usage - _tracked_meta_mem_usage;
+    _tracked_meta_mem_usage = _meta_mem_usage;
 
     return Status::OK();
 }
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index bc5ab1e1fdc80a..fba4871d178bc6 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -57,7 +57,6 @@ class IDataType;
 class ShortKeyIndexDecoder;
 class Schema;
 class StorageReadOptions;
-class MemTracker;
 class PrimaryKeyIndexReader;
 class RowwiseIterator;
 struct RowLocation;
@@ -163,6 +162,8 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
 
     io::FileReaderSPtr file_reader() { return _file_reader; }
 
+    // Including the column reader memory.
+    // another method `get_metadata_size` not include the column reader, only the segment object itself.
     int64_t meta_mem_usage() const { return _meta_mem_usage; }
 
     // Identify the column by unique id or path info
@@ -249,9 +250,8 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
     // 1. Tracking memory use by segment meta data such as footer or index page.
     // 2. Tracking memory use by segment column reader
     // The memory consumed by querying is tracked in segment iterator.
-    // TODO: Segment::_meta_mem_usage Unknown value overflow, causes the value of SegmentMeta mem tracker
-    // is similar to `-2912341218700198079`. So, temporarily put it in experimental type tracker.
     int64_t _meta_mem_usage;
+    int64_t _tracked_meta_mem_usage = 0;
 
     RowsetId _rowset_id;
     TabletSchemaSPtr _tablet_schema;
diff --git a/be/src/olap/rowset_builder.h b/be/src/olap/rowset_builder.h
index 7fd578037363a0..fb2294d1770cc4 100644
--- a/be/src/olap/rowset_builder.h
+++ b/be/src/olap/rowset_builder.h
@@ -38,7 +38,6 @@ namespace doris {
 class CalcDeleteBitmapToken;
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index 26ac54c699b81a..4240f7e250a06b 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -77,9 +77,8 @@ Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
         }
         if (use_cache && !config::disable_segment_cache) {
             // memory of SegmentCache::CacheValue will be handled by SegmentCache
-            auto* cache_value = new SegmentCache::CacheValue();
+            auto* cache_value = new SegmentCache::CacheValue(segment);
             _cache_mem_usage += segment->meta_mem_usage();
-            cache_value->segment = std::move(segment);
             _segment_cache->insert(cache_key, *cache_value, cache_handle);
         } else {
             cache_handle->push_segment(std::move(segment));
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 834906da93bf74..2c5b1ed200dde7 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -75,9 +75,9 @@ class SegmentCache : public LRUCachePolicy {
     // Holding all opened segments of a rowset.
     class CacheValue : public LRUCacheValueBase {
     public:
-        ~CacheValue() override { segment.reset(); }
+        CacheValue(segment_v2::SegmentSharedPtr segment_) : segment(std::move(segment_)) {}
 
-        segment_v2::SegmentSharedPtr segment;
+        const segment_v2::SegmentSharedPtr segment;
     };
 
     SegmentCache(size_t memory_bytes_limit, size_t segment_num_limit)
@@ -124,8 +124,13 @@ class SegmentLoader {
 
     void erase_segments(const RowsetId& rowset_id, int64_t num_segments);
 
-    // Just used for BE UT
-    int64_t cache_mem_usage() const { return _cache_mem_usage; }
+    int64_t cache_mem_usage() const {
+#ifdef BE_TEST
+        return _cache_mem_usage;
+#else
+        return _segment_cache->value_mem_consumption();
+#endif
+    }
 
 private:
     SegmentLoader();
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index d6a944dbc39853..93dbce9126fd52 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -57,8 +57,6 @@
 #include "olap/tablet_schema.h"
 #include "olap/txn_manager.h"
 #include "runtime/exec_env.h"
-#include "runtime/memory/mem_tracker.h"
-#include "runtime/thread_context.h"
 #include "service/backend_options.h"
 #include "util/defer_op.h"
 #include "util/doris_metrics.h"
@@ -83,28 +81,18 @@ using std::vector;
 namespace doris {
 using namespace ErrorCode;
 
-DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_meta_mem_consumption, MetricUnit::BYTES, "",
-                                   mem_consumption, Labels({{"type", "tablet_meta"}}));
-
 bvar::Adder<int64_t> g_tablet_meta_schema_columns_count("tablet_meta_schema_columns_count");
 
 TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size)
         : _engine(engine),
-          _tablet_meta_mem_tracker(std::make_shared<MemTracker>("TabletMeta(experimental)")),
           _tablets_shards_size(tablet_map_lock_shard_size),
           _tablets_shards_mask(tablet_map_lock_shard_size - 1) {
     CHECK_GT(_tablets_shards_size, 0);
     CHECK_EQ(_tablets_shards_size & _tablets_shards_mask, 0);
     _tablets_shards.resize(_tablets_shards_size);
-    REGISTER_HOOK_METRIC(tablet_meta_mem_consumption,
-                         [this]() { return _tablet_meta_mem_tracker->consumption(); });
 }
 
-TabletManager::~TabletManager() {
-#ifndef BE_TEST
-    DEREGISTER_HOOK_METRIC(tablet_meta_mem_consumption);
-#endif
-}
+TabletManager::~TabletManager() = default;
 
 Status TabletManager::_add_tablet_unlocked(TTabletId tablet_id, const TabletSharedPtr& tablet,
                                            bool update_meta, bool force, RuntimeProfile* profile) {
@@ -242,10 +230,6 @@ Status TabletManager::_add_tablet_to_map_unlocked(TTabletId tablet_id,
     tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
     tablet_map[tablet_id] = tablet;
     _add_tablet_to_partition(tablet);
-    // TODO: remove multiply 2 of tablet meta mem size
-    // Because table schema will copy in tablet, there will be double mem cost
-    // so here multiply 2
-    _tablet_meta_mem_tracker->consume(tablet->tablet_meta()->mem_size() * 2);
     g_tablet_meta_schema_columns_count << tablet->tablet_meta()->tablet_columns_num();
     COUNTER_UPDATE(ADD_CHILD_TIMER(profile, "RegisterTabletInfo", "AddTablet"),
                    static_cast<int64_t>(watch.reset()));
@@ -599,7 +583,6 @@ Status TabletManager::_drop_tablet(TTabletId tablet_id, TReplicaId replica_id, b
     }
 
     to_drop_tablet->deregister_tablet_from_dir();
-    _tablet_meta_mem_tracker->release(to_drop_tablet->tablet_meta()->mem_size() * 2);
     g_tablet_meta_schema_columns_count << -to_drop_tablet->tablet_meta()->tablet_columns_num();
     return Status::OK();
 }
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 42623cf05f2aea..6b6e7998f9cee1 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -251,9 +251,6 @@ class TabletManager {
 
     StorageEngine& _engine;
 
-    // TODO: memory size of TabletSchema cannot be accurately tracked.
-    std::shared_ptr<MemTracker> _tablet_meta_mem_tracker;
-
     const int32_t _tablets_shards_size;
     const int32_t _tablets_shards_mask;
     std::vector<tablets_shard> _tablets_shards;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 0570aff349c583..fc9fc034b0bcd7 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -781,12 +781,6 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
             time_series_compaction_level_threshold());
 }
 
-int64_t TabletMeta::mem_size() const {
-    auto size = sizeof(TabletMeta);
-    size += _schema->mem_size();
-    return size;
-}
-
 void TabletMeta::to_json(string* json_string, json2pb::Pb2JsonOptions& options) {
     TabletMetaPB tablet_meta_pb;
     to_meta_pb(&tablet_meta_pb);
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index fb0895604a19fe..5c6556ffb2f937 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -140,10 +140,6 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
 
     void to_meta_pb(TabletMetaPB* tablet_meta_pb);
     void to_json(std::string* json_string, json2pb::Pb2JsonOptions& options);
-    // Don't use.
-    // TODO: memory size of TabletSchema cannot be accurately tracked.
-    // In some places, temporarily use num_columns() as TabletSchema size.
-    int64_t mem_size() const;
     size_t tablet_columns_num() const { return _schema->num_columns(); }
 
     TabletTypePB tablet_type() const { return _tablet_type; }
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index c4f96e2214853d..47179ce19b23e7 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -41,8 +41,6 @@
 #include "olap/tablet_column_object_pool.h"
 #include "olap/types.h"
 #include "olap/utils.h"
-#include "runtime/memory/lru_cache_policy.h"
-#include "runtime/thread_context.h"
 #include "tablet_meta.h"
 #include "vec/aggregate_functions/aggregate_function_simple_factory.h"
 #include "vec/aggregate_functions/aggregate_function_state_union.h"
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 5fb3deafd77319..9a0cd53f7b1786 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -332,10 +332,8 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
     void copy_from(const TabletSchema& tablet_schema);
     void update_index_info_from(const TabletSchema& tablet_schema);
     std::string to_key() const;
-    // Don't use.
-    // TODO: memory size of TabletSchema cannot be accurately tracked.
-    // In some places, temporarily use num_columns() as TabletSchema size.
-    int64_t mem_size() const { return _mem_size; }
+    // get_metadata_size is only the memory of the TabletSchema itself, not include child objects.
+    int64_t mem_size() const { return get_metadata_size(); }
     size_t row_size() const;
     int32_t field_index(const std::string& field_name) const;
     int32_t field_index(const vectorized::PathInData& path) const;
@@ -573,7 +571,6 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
     int64_t _db_id = -1;
     bool _disable_auto_compaction = false;
     bool _enable_single_replica_compaction = false;
-    int64_t _mem_size = 0;
     bool _store_row_column = false;
     bool _skip_write_index_on_load = false;
     InvertedIndexStorageFormatPB _inverted_index_storage_format = InvertedIndexStorageFormatPB::V1;
diff --git a/be/src/olap/tablet_schema_cache.cpp b/be/src/olap/tablet_schema_cache.cpp
index fd238fa5affb3f..e044ef9c0426f4 100644
--- a/be/src/olap/tablet_schema_cache.cpp
+++ b/be/src/olap/tablet_schema_cache.cpp
@@ -56,7 +56,7 @@ std::pair<Cache::Handle*, TabletSchemaSPtr> TabletSchemaCache::insert(const std:
         tablet_schema_ptr->init_from_pb(pb, false, true);
         value->tablet_schema = tablet_schema_ptr;
         lru_handle = LRUCachePolicy::insert(key_signature, value, tablet_schema_ptr->num_columns(),
-                                            0, CachePriority::NORMAL);
+                                            tablet_schema_ptr->mem_size(), CachePriority::NORMAL);
         g_tablet_schema_cache_count << 1;
         g_tablet_schema_cache_columns_count << tablet_schema_ptr->num_columns();
     }
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index a27936f5f0d88b..636ce2bf288b58 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -178,7 +178,6 @@ class ExecEnv {
     std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool;
     void init_mem_tracker();
     std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; }
-    std::shared_ptr<MemTracker> page_no_cache_mem_tracker() { return _page_no_cache_mem_tracker; }
     std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() {
         return _brpc_iobuf_block_memory_tracker;
     }
@@ -188,6 +187,15 @@ class ExecEnv {
     std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() {
         return _stream_load_pipe_tracker;
     }
+    std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() {
+        return _tablets_no_cache_mem_tracker;
+    }
+    std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() {
+        return _rowsets_no_cache_mem_tracker;
+    }
+    std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() {
+        return _segments_no_cache_mem_tracker;
+    }
     std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() {
         return _point_query_executor_mem_tracker;
     }
@@ -377,13 +385,15 @@ class ExecEnv {
     // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership",
     // and the consumption of the orphan mem tracker is close to 0, but greater than 0.
     std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker;
-    // page size not in cache, data page/index page/etc.
-    std::shared_ptr<MemTracker> _page_no_cache_mem_tracker;
     std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker;
     // Count the memory consumption of segment compaction tasks.
     std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker;
     std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker;
 
+    std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker;
+    std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker;
+    std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker;
+
     // Tracking memory may be shared between multiple queries.
     std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker;
     std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index b7f926cc3b4512..a371cdb947ff56 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -599,15 +599,20 @@ void ExecEnv::init_mem_tracker() {
     _s_tracking_memory = true;
     _orphan_mem_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "Orphan");
-    _page_no_cache_mem_tracker = std::make_shared<MemTracker>("PageNoCache");
     _brpc_iobuf_block_memory_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "IOBufBlockMemory");
     _segcompaction_mem_tracker =
-            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "SegCompaction");
+            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::COMPACTION, "SegCompaction");
+    _tablets_no_cache_mem_tracker = MemTrackerLimiter::create_shared(
+            MemTrackerLimiter::Type::METADATA, "Tablets(not in SchemaCache, TabletSchemaCache)");
+    _segments_no_cache_mem_tracker = MemTrackerLimiter::create_shared(
+            MemTrackerLimiter::Type::METADATA, "Segments(not in SegmentCache)");
+    _rowsets_no_cache_mem_tracker =
+            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::METADATA, "Rowsets");
     _point_query_executor_mem_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "PointQueryExecutor");
     _query_cache_mem_tracker =
-            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "QueryCache");
+            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::CACHE, "QueryCache");
     _block_compression_mem_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "BlockCompression");
     _rowid_storage_reader_tracker =
@@ -716,7 +721,7 @@ void ExecEnv::destroy() {
     _file_cache_open_fd_cache.reset();
     SAFE_STOP(_write_cooldown_meta_executors);
 
-    // StorageEngine must be destoried before _page_no_cache_mem_tracker.reset and _cache_manager destory
+    // StorageEngine must be destoried before _cache_manager destory
     SAFE_STOP(_storage_engine);
     _storage_engine.reset();
 
diff --git a/be/src/runtime/memory/cache_policy.h b/be/src/runtime/memory/cache_policy.h
index e7e1c73e7cbb41..8f077a4eb45bb1 100644
--- a/be/src/runtime/memory/cache_policy.h
+++ b/be/src/runtime/memory/cache_policy.h
@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <vector>
+
 #include "util/runtime_profile.h"
 
 namespace doris {
@@ -123,6 +125,7 @@ class CachePolicy {
             {"CloudTabletCache", CacheType::CLOUD_TABLET_CACHE},
             {"CloudTxnDeleteBitmapCache", CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE},
             {"ForUTCacheNumber", CacheType::FOR_UT_CACHE_NUMBER},
+            {"QueryCache", CacheType::QUERY_CACHE},
             {"TabletColumnObjectPool", CacheType::TABLET_COLUMN_OBJECT_POOL}};
 
     static CacheType string_to_type(std::string type) {
@@ -133,6 +136,9 @@ class CachePolicy {
         }
     }
 
+    inline static std::vector<CacheType> MetadataCache {
+            CacheType::SEGMENT_CACHE, CacheType::SCHEMA_CACHE, CacheType::TABLET_SCHEMA_CACHE};
+
     CachePolicy(CacheType type, size_t capacity, uint32_t stale_sweep_time_s, bool enable_prune);
     virtual ~CachePolicy();
 
diff --git a/be/src/runtime/memory/lru_cache_policy.h b/be/src/runtime/memory/lru_cache_policy.h
index ea34e2837f1313..3fdb43facd7715 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -104,20 +104,26 @@ class LRUCachePolicy : public CachePolicy {
         return _mem_tracker->consumption();
     }
 
+    int64_t value_mem_consumption() {
+        DCHECK(_value_mem_tracker != nullptr);
+        return _value_mem_tracker->consumption();
+    }
+
     // Insert will consume tracking_bytes to _mem_tracker and cache value destroy will release tracking_bytes.
-    // If LRUCacheType::SIZE, tracking_bytes usually equal to charge.
-    // If LRUCacheType::NUMBER, tracking_bytes usually not equal to charge, at this time charge is an weight.
-    // If LRUCacheType::SIZE and tracking_bytes equals 0, memory must be tracked in Doris Allocator,
+    // If LRUCacheType::SIZE, value_tracking_bytes usually equal to charge.
+    // If LRUCacheType::NUMBER, value_tracking_bytes usually not equal to charge, at this time charge is an weight.
+    // If LRUCacheType::SIZE and value_tracking_bytes equals 0, memory must be tracked in Doris Allocator,
     //    cache value is allocated using Alloctor.
-    // If LRUCacheType::NUMBER and tracking_bytes equals 0, usually currently cannot accurately tracking memory size,
+    // If LRUCacheType::NUMBER and value_tracking_bytes equals 0, usually currently cannot accurately tracking memory size,
     //    only tracking handle_size(106).
-    Cache::Handle* insert(const CacheKey& key, void* value, size_t charge, size_t tracking_bytes,
+    Cache::Handle* insert(const CacheKey& key, void* value, size_t charge,
+                          size_t value_tracking_bytes,
                           CachePriority priority = CachePriority::NORMAL) {
-        size_t tracking_bytes_with_handle = sizeof(LRUHandle) - 1 + key.size() + tracking_bytes;
+        size_t tracking_bytes = sizeof(LRUHandle) - 1 + key.size() + value_tracking_bytes;
         if (value != nullptr) {
-            mem_tracker()->consume(tracking_bytes_with_handle);
             ((LRUCacheValueBase*)value)
-                    ->set_tracking_bytes(tracking_bytes_with_handle, _mem_tracker);
+                    ->set_tracking_bytes(tracking_bytes, _mem_tracker, value_tracking_bytes,
+                                         _value_mem_tracker);
         }
         return _cache->insert(key, value, charge, priority);
     }
@@ -265,9 +271,18 @@ class LRUCachePolicy : public CachePolicy {
 
 protected:
     void _init_mem_tracker(const std::string& type_name) {
-        _mem_tracker = MemTrackerLimiter::create_shared(
-                MemTrackerLimiter::Type::GLOBAL,
-                fmt::format("{}[{}]", type_string(_type), type_name));
+        if (std::find(CachePolicy::MetadataCache.begin(), CachePolicy::MetadataCache.end(),
+                      _type) == CachePolicy::MetadataCache.end()) {
+            _mem_tracker = MemTrackerLimiter::create_shared(
+                    MemTrackerLimiter::Type::CACHE,
+                    fmt::format("{}[{}]", type_string(_type), type_name));
+        } else {
+            _mem_tracker = MemTrackerLimiter::create_shared(
+                    MemTrackerLimiter::Type::METADATA,
+                    fmt::format("{}[{}]", type_string(_type), type_name));
+        }
+        _value_mem_tracker = std::make_shared<MemTracker>(
+                fmt::format("{}::Value[{}]", type_string(_type), type_name));
     }
 
     // if check_capacity failed, will return dummy lru cache,
@@ -277,6 +292,7 @@ class LRUCachePolicy : public CachePolicy {
     LRUCacheType _lru_cache_type;
 
     std::shared_ptr<MemTrackerLimiter> _mem_tracker;
+    std::shared_ptr<MemTracker> _value_mem_tracker;
 };
 
 } // namespace doris
diff --git a/be/src/runtime/memory/lru_cache_value_base.h b/be/src/runtime/memory/lru_cache_value_base.h
index f9e534e6600df8..a9a3ae5ddab632 100644
--- a/be/src/runtime/memory/lru_cache_value_base.h
+++ b/be/src/runtime/memory/lru_cache_value_base.h
@@ -28,18 +28,27 @@ class LRUCacheValueBase {
     virtual ~LRUCacheValueBase() {
         if (_tracking_bytes > 0) {
             _mem_tracker->release(_tracking_bytes);
+            _value_mem_tracker->release(_value_tracking_bytes);
         }
     }
 
     void set_tracking_bytes(size_t tracking_bytes,
-                            const std::shared_ptr<MemTrackerLimiter>& mem_tracker) {
+                            const std::shared_ptr<MemTrackerLimiter>& mem_tracker,
+                            size_t value_tracking_bytes,
+                            const std::shared_ptr<MemTracker>& value_mem_tracker) {
         this->_tracking_bytes = tracking_bytes;
         this->_mem_tracker = mem_tracker;
+        this->_value_tracking_bytes = value_tracking_bytes;
+        this->_value_mem_tracker = value_mem_tracker;
+        _mem_tracker->consume(_tracking_bytes);
+        _value_mem_tracker->consume(_value_tracking_bytes);
     }
 
 protected:
     size_t _tracking_bytes = 0;
+    size_t _value_tracking_bytes = 0;
     std::shared_ptr<MemTrackerLimiter> _mem_tracker;
+    std::shared_ptr<MemTracker> _value_mem_tracker;
 };
 
 } // namespace doris
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 05ff13f0e7c646..25d5d307b5bb99 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -66,9 +66,13 @@ MemTrackerLimiter::MemTrackerLimiter(Type type, const std::string& label, int64_
     _uid = UniqueId::gen_uid();
     if (_type == Type::GLOBAL) {
         _group_num = 0;
+    } else if (_type == Type::METADATA) {
+        _group_num = 1;
+    } else if (_type == Type::CACHE) {
+        _group_num = 2;
     } else {
         _group_num =
-                mem_tracker_limiter_group_counter.fetch_add(1) % (MEM_TRACKER_GROUP_NUM - 1) + 1;
+                mem_tracker_limiter_group_counter.fetch_add(1) % (MEM_TRACKER_GROUP_NUM - 3) + 3;
     }
 
     // currently only select/load need runtime query statistics
@@ -268,8 +272,26 @@ void MemTrackerLimiter::make_type_trackers_profile(RuntimeProfile* profile,
                 tracker->make_profile(profile);
             }
         }
+    } else if (type == Type::METADATA) {
+        std::lock_guard<std::mutex> l(
+                ExecEnv::GetInstance()->mem_tracker_limiter_pool[1].group_lock);
+        for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[1].trackers) {
+            auto tracker = trackerWptr.lock();
+            if (tracker != nullptr) {
+                tracker->make_profile(profile);
+            }
+        }
+    } else if (type == Type::CACHE) {
+        std::lock_guard<std::mutex> l(
+                ExecEnv::GetInstance()->mem_tracker_limiter_pool[2].group_lock);
+        for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[2].trackers) {
+            auto tracker = trackerWptr.lock();
+            if (tracker != nullptr) {
+                tracker->make_profile(profile);
+            }
+        }
     } else {
-        for (unsigned i = 1; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
+        for (unsigned i = 3; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
             std::lock_guard<std::mutex> l(
                     ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].group_lock);
             for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].trackers) {
@@ -296,8 +318,8 @@ void MemTrackerLimiter::make_top_consumption_tasks_tracker_profile(RuntimeProfil
     std::unique_ptr<RuntimeProfile> tmp_profile_snapshot =
             std::make_unique<RuntimeProfile>("tmpSnapshot");
     std::priority_queue<std::pair<int64_t, RuntimeProfile*>> max_pq;
-    // start from 2, not include global type.
-    for (unsigned i = 1; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
+    // start from 3, not include global/metadata/cache type.
+    for (unsigned i = 3; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
         std::lock_guard<std::mutex> l(
                 ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].group_lock);
         for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].trackers) {
@@ -326,13 +348,19 @@ void MemTrackerLimiter::make_all_tasks_tracker_profile(RuntimeProfile* profile)
     types_profile[Type::SCHEMA_CHANGE] = profile->create_child("SchemaChangeTasks", true, false);
     types_profile[Type::OTHER] = profile->create_child("OtherTasks", true, false);
 
-    // start from 2, not include global type.
-    for (unsigned i = 1; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
+    // start from 3, not include global/metadata/cache type.
+    for (unsigned i = 3; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
         std::lock_guard<std::mutex> l(
                 ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].group_lock);
         for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].trackers) {
             auto tracker = trackerWptr.lock();
             if (tracker != nullptr) {
+                // BufferControlBlock will continue to exist for 5 minutes after the query ends, even if the
+                // result buffer is empty, and will not be shown in the profile. of course, this code is tricky.
+                if (tracker->consumption() == 0 &&
+                    tracker->label().starts_with("BufferControlBlock")) {
+                    continue;
+                }
                 tracker->make_profile(types_profile[tracker->type()]);
             }
         }
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h
index 445856b1f6af83..43b20a410ff27c 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -77,12 +77,14 @@ class MemTrackerLimiter final {
     enum class GCType { PROCESS = 0, WORK_LOAD_GROUP = 1 };
 
     enum class Type {
-        GLOBAL = 0,        // Life cycle is the same as the process, e.g. Cache and default Orphan
+        GLOBAL = 0,        // Life cycle is the same as the process, except cache and metadata.
         QUERY = 1,         // Count the memory consumption of all Query tasks.
         LOAD = 2,          // Count the memory consumption of all Load tasks.
         COMPACTION = 3,    // Count the memory consumption of all Base and Cumulative tasks.
         SCHEMA_CHANGE = 4, // Count the memory consumption of all SchemaChange tasks.
-        OTHER = 5,
+        METADATA = 5,      // Count the memory consumption of all Metadata.
+        CACHE = 6,         // Count the memory consumption of all Cache.
+        OTHER = 7, // Count the memory consumption of all other tasks, such as Clone, Snapshot, etc..
     };
 
     static std::string type_string(Type type) {
@@ -97,8 +99,12 @@ class MemTrackerLimiter final {
             return "compaction";
         case Type::SCHEMA_CHANGE:
             return "schema_change";
+        case Type::METADATA:
+            return "metadata";
+        case Type::CACHE:
+            return "cache";
         case Type::OTHER:
-            return "other";
+            return "other_task";
         default:
             LOG(FATAL) << "not match type of mem tracker limiter :" << static_cast<int>(type);
         }
@@ -158,6 +164,8 @@ class MemTrackerLimiter final {
     int64_t consumption() const { return _mem_counter.current_value(); }
     int64_t peak_consumption() const { return _mem_counter.peak_value(); }
 
+    // Use carefully! only memory that cannot be allocated using Doris Allocator needs to be consumed manually.
+    // Ideally, all memory should use Doris Allocator.
     void consume(int64_t bytes) {
         _mem_counter.add(bytes);
         if (_query_statistics) {
diff --git a/be/src/runtime/memory/memory_profile.cpp b/be/src/runtime/memory/memory_profile.cpp
index 8dbdcbdd3af769..651a6a6cefaec1 100644
--- a/be/src/runtime/memory/memory_profile.cpp
+++ b/be/src/runtime/memory/memory_profile.cpp
@@ -18,6 +18,9 @@
 #include "runtime/memory/memory_profile.h"
 
 #include "bvar/reducer.h"
+#include "olap/metadata_adder.h"
+#include "olap/schema_cache.h"
+#include "olap/tablet_schema_cache.h"
 #include "runtime/exec_env.h"
 #include "runtime/memory/global_memory_arbitrator.h"
 #include "runtime/memory/mem_tracker_limiter.h"
@@ -28,6 +31,9 @@ namespace doris {
 
 static bvar::Adder<int64_t> memory_all_tracked_sum_bytes("memory_all_tracked_sum_bytes");
 static bvar::Adder<int64_t> memory_global_trackers_sum_bytes("memory_global_trackers_sum_bytes");
+static bvar::Adder<int64_t> memory_metadata_trackers_sum_bytes(
+        "memory_metadata_trackers_sum_bytes");
+static bvar::Adder<int64_t> memory_cache_trackers_sum_bytes("memory_cache_trackers_sum_bytes");
 static bvar::Adder<int64_t> memory_query_trackers_sum_bytes("memory_query_trackers_sum_bytes");
 static bvar::Adder<int64_t> memory_load_trackers_sum_bytes("memory_load_trackers_sum_bytes");
 static bvar::Adder<int64_t> memory_compaction_trackers_sum_bytes(
@@ -42,6 +48,8 @@ static bvar::Adder<int64_t> memory_untracked_memory_bytes("memory_untracked_memo
 MemoryProfile::MemoryProfile() {
     _memory_overview_profile.set(std::make_unique<RuntimeProfile>("MemoryOverviewSnapshot"));
     _global_memory_profile.set(std::make_unique<RuntimeProfile>("GlobalMemorySnapshot"));
+    _metadata_memory_profile.set(std::make_unique<RuntimeProfile>("MetadataMemorySnapshot"));
+    _cache_memory_profile.set(std::make_unique<RuntimeProfile>("CacheMemorySnapshot"));
     _top_memory_tasks_profile.set(std::make_unique<RuntimeProfile>("TopMemoryTasksSnapshot"));
     _tasks_memory_profile.set(std::make_unique<RuntimeProfile>("TasksMemorySnapshot"));
 }
@@ -56,6 +64,10 @@ void MemoryProfile::refresh_memory_overview_profile() {
 #endif
     std::unique_ptr<RuntimeProfile> global_memory_profile =
             std::make_unique<RuntimeProfile>("GlobalMemorySnapshot");
+    std::unique_ptr<RuntimeProfile> metadata_memory_profile =
+            std::make_unique<RuntimeProfile>("MetadataMemorySnapshot");
+    std::unique_ptr<RuntimeProfile> cache_memory_profile =
+            std::make_unique<RuntimeProfile>("CacheMemorySnapshot");
     std::unique_ptr<RuntimeProfile> top_memory_tasks_profile =
             std::make_unique<RuntimeProfile>("TopMemoryTasksSnapshot");
 
@@ -70,6 +82,10 @@ void MemoryProfile::refresh_memory_overview_profile() {
             tasks_memory_overview_profile->create_child("Details", true, false);
     RuntimeProfile* global_memory_overview_profile =
             tracked_memory_profile->create_child("GlobalMemory", true, false);
+    RuntimeProfile* metadata_memory_overview_profile =
+            tracked_memory_profile->create_child("MetadataMemory", true, false);
+    RuntimeProfile* cache_memory_overview_profile =
+            tracked_memory_profile->create_child("CacheMemory", true, false);
     RuntimeProfile* jemalloc_memory_profile =
             tracked_memory_profile->create_child("JemallocMemory", true, false);
     RuntimeProfile* jemalloc_memory_details_profile =
@@ -112,11 +128,19 @@ void MemoryProfile::refresh_memory_overview_profile() {
     RuntimeProfile::Counter* jemalloc_metadata_peak_usage_counter =
             jemalloc_memory_details_profile->AddHighWaterMarkCounter("MetadataPeak", TUnit::BYTES);
 
-    // 2.5 add global memory counter
+    // 2.5 add global/metadata/cache memory counter
     RuntimeProfile::Counter* global_current_usage_counter =
             ADD_COUNTER(global_memory_overview_profile, "CurrentUsage", TUnit::BYTES);
     RuntimeProfile::Counter* global_peak_usage_counter =
             global_memory_overview_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
+    RuntimeProfile::Counter* metadata_current_usage_counter =
+            ADD_COUNTER(metadata_memory_overview_profile, "CurrentUsage", TUnit::BYTES);
+    RuntimeProfile::Counter* metadata_peak_usage_counter =
+            metadata_memory_overview_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
+    RuntimeProfile::Counter* cache_current_usage_counter =
+            ADD_COUNTER(cache_memory_overview_profile, "CurrentUsage", TUnit::BYTES);
+    RuntimeProfile::Counter* cache_peak_usage_counter =
+            cache_memory_overview_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
 
     // 2.6 add tasks memory counter
     RuntimeProfile::Counter* tasks_memory_current_usage_counter =
@@ -169,11 +193,23 @@ void MemoryProfile::refresh_memory_overview_profile() {
                 PerfCounters::get_vm_size()); // from /proc VmSize VmPeak
     COUNTER_SET(process_virtual_memory_peak_usage_counter, PerfCounters::get_vm_peak());
 
-    // 3.2 refresh tracked memory counter
+    // 3.2 refresh metadata memory tracker
+    ExecEnv::GetInstance()->tablets_no_cache_mem_tracker()->set_consumption(
+            MetadataAdder<TabletMeta>::get_all_tablets_size() -
+            TabletSchemaCache::instance()->value_mem_consumption() -
+            SchemaCache::instance()->value_mem_consumption());
+    ExecEnv::GetInstance()->rowsets_no_cache_mem_tracker()->set_consumption(
+            MetadataAdder<RowsetMeta>::get_all_rowsets_size());
+    ExecEnv::GetInstance()->segments_no_cache_mem_tracker()->set_consumption(
+            MetadataAdder<segment_v2::Segment>::get_all_segments_estimate_size() -
+            SegmentLoader::instance()->cache_mem_usage());
+
+    // 3.3 refresh tracked memory counter
     std::unordered_map<MemTrackerLimiter::Type, int64_t> type_mem_sum = {
             {MemTrackerLimiter::Type::GLOBAL, 0},        {MemTrackerLimiter::Type::QUERY, 0},
             {MemTrackerLimiter::Type::LOAD, 0},          {MemTrackerLimiter::Type::COMPACTION, 0},
-            {MemTrackerLimiter::Type::SCHEMA_CHANGE, 0}, {MemTrackerLimiter::Type::OTHER, 0}};
+            {MemTrackerLimiter::Type::SCHEMA_CHANGE, 0}, {MemTrackerLimiter::Type::METADATA, 0},
+            {MemTrackerLimiter::Type::CACHE, 0},         {MemTrackerLimiter::Type::OTHER, 0}};
     // always ExecEnv::ready(), because Daemon::_stop_background_threads_latch
     for (auto& group : ExecEnv::GetInstance()->mem_tracker_limiter_pool) {
         std::lock_guard<std::mutex> l(group.group_lock);
@@ -224,6 +260,18 @@ void MemoryProfile::refresh_memory_overview_profile() {
             memory_schema_change_trackers_sum_bytes
                     << it.second - memory_schema_change_trackers_sum_bytes.get_value();
             break;
+        case MemTrackerLimiter::Type::METADATA:
+            COUNTER_SET(metadata_current_usage_counter, it.second);
+            COUNTER_SET(metadata_peak_usage_counter, it.second);
+            memory_metadata_trackers_sum_bytes
+                    << it.second - memory_metadata_trackers_sum_bytes.get_value();
+            break;
+        case MemTrackerLimiter::Type::CACHE:
+            COUNTER_SET(cache_current_usage_counter, it.second);
+            COUNTER_SET(cache_peak_usage_counter, it.second);
+            memory_cache_trackers_sum_bytes
+                    << it.second - memory_cache_trackers_sum_bytes.get_value();
+            break;
         case MemTrackerLimiter::Type::OTHER:
             COUNTER_SET(other_current_usage_counter, it.second);
             COUNTER_SET(other_peak_usage_counter, it.second);
@@ -235,6 +283,10 @@ void MemoryProfile::refresh_memory_overview_profile() {
 
     MemTrackerLimiter::make_type_trackers_profile(global_memory_profile.get(),
                                                   MemTrackerLimiter::Type::GLOBAL);
+    MemTrackerLimiter::make_type_trackers_profile(metadata_memory_profile.get(),
+                                                  MemTrackerLimiter::Type::METADATA);
+    MemTrackerLimiter::make_type_trackers_profile(cache_memory_profile.get(),
+                                                  MemTrackerLimiter::Type::CACHE);
 
     MemTrackerLimiter::make_top_consumption_tasks_tracker_profile(top_memory_tasks_profile.get(),
                                                                   15);
@@ -272,14 +324,14 @@ void MemoryProfile::refresh_memory_overview_profile() {
     COUNTER_SET(tracked_memory_peak_usage_counter, all_tracked_mem_sum);
     memory_all_tracked_sum_bytes << all_tracked_mem_sum - memory_all_tracked_sum_bytes.get_value();
 
-    // 3.3 refresh untracked memory counter
+    // 3.4 refresh untracked memory counter
     int64_t untracked_memory =
             process_physical_memory_current_usage_counter->value() - all_tracked_mem_sum;
     COUNTER_SET(untracked_memory_current_usage_counter, untracked_memory);
     COUNTER_SET(untracked_memory_peak_usage_counter, untracked_memory);
     memory_untracked_memory_bytes << untracked_memory - memory_untracked_memory_bytes.get_value();
 
-    // 3.4 refresh additional tracker printed when memory exceeds limit.
+    // 3.5 refresh additional tracker printed when memory exceeds limit.
     COUNTER_SET(load_all_memtables_current_usage_counter,
                 ExecEnv::GetInstance()->memtable_memory_limiter()->mem_tracker()->consumption());
     COUNTER_SET(
@@ -289,6 +341,8 @@ void MemoryProfile::refresh_memory_overview_profile() {
     // 4. reset profile
     _memory_overview_profile.set(std::move(memory_overview_profile));
     _global_memory_profile.set(std::move(global_memory_profile));
+    _metadata_memory_profile.set(std::move(metadata_memory_profile));
+    _cache_memory_profile.set(std::move(cache_memory_profile));
     _top_memory_tasks_profile.set(std::move(top_memory_tasks_profile));
 }
 
@@ -312,6 +366,16 @@ void MemoryProfile::make_memory_profile(RuntimeProfile* profile) const {
             memory_profile_snapshot->create_child(global_memory_version_ptr->name(), true, false);
     global_memory_profile->merge(const_cast<RuntimeProfile*>(global_memory_version_ptr.get()));
 
+    auto metadata_memory_version_ptr = _metadata_memory_profile.get();
+    RuntimeProfile* metadata_memory_profile =
+            memory_profile_snapshot->create_child(metadata_memory_version_ptr->name(), true, false);
+    metadata_memory_profile->merge(const_cast<RuntimeProfile*>(metadata_memory_version_ptr.get()));
+
+    auto cache_memory_version_ptr = _cache_memory_profile.get();
+    RuntimeProfile* cache_memory_profile =
+            memory_profile_snapshot->create_child(cache_memory_version_ptr->name(), true, false);
+    cache_memory_profile->merge(const_cast<RuntimeProfile*>(cache_memory_version_ptr.get()));
+
     auto top_memory_tasks_version_ptr = _top_memory_tasks_profile.get();
     RuntimeProfile* top_memory_tasks_profile = memory_profile_snapshot->create_child(
             top_memory_tasks_version_ptr->name(), true, false);
@@ -346,6 +410,8 @@ void MemoryProfile::print_log_process_usage() {
         LOG(WARNING) << "Process Memory Summary: " + GlobalMemoryArbitrator::process_mem_log_str();
         LOG(WARNING) << "\n" << print_memory_overview_profile();
         LOG(WARNING) << "\n" << print_global_memory_profile();
+        LOG(WARNING) << "\n" << print_metadata_memory_profile();
+        LOG(WARNING) << "\n" << print_cache_memory_profile();
         LOG(WARNING) << "\n" << print_top_memory_tasks_profile();
     }
 }
diff --git a/be/src/runtime/memory/memory_profile.h b/be/src/runtime/memory/memory_profile.h
index 9f1bab0c02a802..6d301a0be8b05c 100644
--- a/be/src/runtime/memory/memory_profile.h
+++ b/be/src/runtime/memory/memory_profile.h
@@ -33,31 +33,27 @@ class MemoryProfile {
     void make_memory_profile(RuntimeProfile* profile) const;
 
     std::string print_memory_overview_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _memory_overview_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_memory_overview_profile.get());
     }
 
     std::string print_global_memory_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _global_memory_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_global_memory_profile.get());
+    }
+
+    std::string print_metadata_memory_profile() const {
+        return return_memory_profile_str(_metadata_memory_profile.get());
+    }
+
+    std::string print_cache_memory_profile() const {
+        return return_memory_profile_str(_cache_memory_profile.get());
     }
 
     std::string print_top_memory_tasks_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _top_memory_tasks_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_top_memory_tasks_profile.get());
     }
 
     std::string print_tasks_memory_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _tasks_memory_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_tasks_memory_profile.get());
     }
 
     static int64_t query_current_usage();
@@ -71,8 +67,17 @@ class MemoryProfile {
     void print_log_process_usage();
 
 private:
+    std::string return_memory_profile_str(
+            const std::shared_ptr<const RuntimeProfile>& profile) const {
+        std::stringstream ss;
+        profile->pretty_print(&ss);
+        return ss.str();
+    }
+
     MultiVersion<RuntimeProfile> _memory_overview_profile;
     MultiVersion<RuntimeProfile> _global_memory_profile;
+    MultiVersion<RuntimeProfile> _metadata_memory_profile;
+    MultiVersion<RuntimeProfile> _cache_memory_profile;
     MultiVersion<RuntimeProfile> _top_memory_tasks_profile;
     MultiVersion<RuntimeProfile> _tasks_memory_profile;
 
diff --git a/be/src/vec/runtime/partitioner.h b/be/src/vec/runtime/partitioner.h
index 39ed0e899411b8..d5492c3be87683 100644
--- a/be/src/vec/runtime/partitioner.h
+++ b/be/src/vec/runtime/partitioner.h
@@ -23,7 +23,6 @@
 
 namespace doris {
 #include "common/compile_check_begin.h"
-class MemTracker;
 
 namespace vectorized {
 
diff --git a/be/src/vec/sink/vdata_stream_sender.h b/be/src/vec/sink/vdata_stream_sender.h
index 5fe35e4da119d0..4999602fdf49a7 100644
--- a/be/src/vec/sink/vdata_stream_sender.h
+++ b/be/src/vec/sink/vdata_stream_sender.h
@@ -56,7 +56,6 @@ namespace doris {
 #include "common/compile_check_begin.h"
 class ObjectPool;
 class RuntimeState;
-class MemTracker;
 class RowDescriptor;
 class TDataSink;
 class TDataStreamSink;