diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h index bdc9e7a398d8a3..559c5db873b2f6 100644 --- a/be/src/olap/metadata_adder.h +++ b/be/src/olap/metadata_adder.h @@ -20,11 +20,16 @@ #include #include +#include "util/runtime_profile.h" + namespace doris { inline bvar::Adder g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes"); inline bvar::Adder g_rowset_meta_num("doris_rowset_meta_num"); +inline bvar::Adder g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes"); +inline bvar::Adder g_all_rowsets_num("doris_all_rowsets_num"); + inline bvar::Adder g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes"); inline bvar::Adder g_tablet_meta_num("doris_tablet_meta_num"); @@ -37,8 +42,8 @@ inline bvar::Adder g_tablet_index_num("doris_tablet_index_num"); inline bvar::Adder g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes"); inline bvar::Adder g_tablet_schema_num("doris_tablet_schema_num"); -inline bvar::Adder g_segment_mem_bytes("doris_segment_mem_bytes"); -inline bvar::Adder g_segment_num("doris_segment_num"); +inline bvar::Adder g_all_segments_mem_bytes("doris_all_segments_mem_bytes"); +inline bvar::Adder g_all_segments_num("doris_all_segments_num"); inline bvar::Adder g_column_reader_mem_bytes("doris_column_reader_mem_bytes"); inline bvar::Adder g_column_reader_num("doris_column_reader_num"); @@ -104,6 +109,8 @@ class MetadataAdder { public: MetadataAdder(); + static void dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot); + protected: MetadataAdder(const MetadataAdder& other); @@ -159,6 +166,8 @@ void MetadataAdder::add_mem_size(int64_t val) { } if constexpr (std::is_same_v) { g_rowset_meta_mem_bytes << val; + } else if constexpr (std::is_same_v) { + g_all_rowsets_mem_bytes << val; } else if constexpr (std::is_same_v) { g_tablet_meta_mem_bytes << val; } else if constexpr (std::is_same_v) { @@ -168,7 +177,7 @@ void MetadataAdder::add_mem_size(int64_t val) { } else if constexpr (std::is_same_v) { g_tablet_schema_mem_bytes << val; } else if constexpr (std::is_same_v) { - g_segment_mem_bytes << val; + g_all_segments_mem_bytes << val; } else if constexpr (std::is_same_v) { g_column_reader_mem_bytes << val; } else if constexpr (std::is_same_v) { @@ -185,6 +194,9 @@ void MetadataAdder::add_mem_size(int64_t val) { g_ordinal_index_reader_mem_bytes << val; } else if constexpr (std::is_same_v) { g_zone_map_index_reader_mem_bytes << val; + } else { + LOG(FATAL) << "add_mem_size not match class type: " << typeid(T).name() << ", " << val; + __builtin_unreachable(); } } @@ -195,6 +207,8 @@ void MetadataAdder::add_num(int64_t val) { } if constexpr (std::is_same_v) { g_rowset_meta_num << val; + } else if constexpr (std::is_same_v) { + g_all_rowsets_num << val; } else if constexpr (std::is_same_v) { g_tablet_meta_num << val; } else if constexpr (std::is_same_v) { @@ -204,7 +218,7 @@ void MetadataAdder::add_num(int64_t val) { } else if constexpr (std::is_same_v) { g_tablet_schema_num << val; } else if constexpr (std::is_same_v) { - g_segment_num << val; + g_all_segments_num << val; } else if constexpr (std::is_same_v) { g_column_reader_num << val; } else if constexpr (std::is_same_v) { @@ -221,7 +235,123 @@ void MetadataAdder::add_num(int64_t val) { g_ordinal_index_reader_num << val; } else if constexpr (std::is_same_v) { g_zone_map_index_reader_num << val; + } else { + LOG(FATAL) << "add_num not match class type: " << typeid(T).name() << ", " << val; + __builtin_unreachable(); } } -}; // namespace doris \ No newline at end of file +template +void MetadataAdder::dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot) { + RuntimeProfile::Counter* rowset_meta_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* rowset_meta_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaNum", TUnit::UNIT); + COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value()); + COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value()); + + RuntimeProfile::Counter* all_rowsets_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* all_rowsets_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT); + COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value()); + COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value()); + + RuntimeProfile::Counter* tablet_meta_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* tablet_meta_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaNum", TUnit::UNIT); + COUNTER_SET(tablet_meta_mem_bytes_counter, g_tablet_meta_mem_bytes.get_value()); + COUNTER_SET(tablet_meta_num_counter, g_tablet_meta_num.get_value()); + + RuntimeProfile::Counter* tablet_column_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* tablet_column_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnNum", TUnit::UNIT); + COUNTER_SET(tablet_column_mem_bytes_counter, g_tablet_column_mem_bytes.get_value()); + COUNTER_SET(tablet_column_num_counter, g_tablet_column_num.get_value()); + + RuntimeProfile::Counter* tablet_index_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* tablet_index_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexNum", TUnit::UNIT); + COUNTER_SET(tablet_index_mem_bytes_counter, g_tablet_index_mem_bytes.get_value()); + COUNTER_SET(tablet_index_num_counter, g_tablet_index_num.get_value()); + + RuntimeProfile::Counter* tablet_schema_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* tablet_schema_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaNum", TUnit::UNIT); + COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value()); + COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value()); + + RuntimeProfile::Counter* all_segments_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* all_segments_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT); + COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value()); + COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value()); + + RuntimeProfile::Counter* column_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* column_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderNum", TUnit::UNIT); + COUNTER_SET(column_reader_mem_bytes_counter, g_column_reader_mem_bytes.get_value()); + COUNTER_SET(column_reader_num_counter, g_column_reader_num.get_value()); + + RuntimeProfile::Counter* bitmap_index_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* bitmap_index_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderNum", TUnit::UNIT); + COUNTER_SET(bitmap_index_reader_mem_bytes_counter, g_bitmap_index_reader_mem_bytes.get_value()); + COUNTER_SET(bitmap_index_reader_num_counter, g_bitmap_index_reader_num.get_value()); + + RuntimeProfile::Counter* bloom_filter_index_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* filter_index_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderNum", TUnit::UNIT); + COUNTER_SET(bloom_filter_index_reader_mem_bytes_counter, + g_bloom_filter_index_reader_mem_bytes.get_value()); + COUNTER_SET(filter_index_reader_num_counter, g_bloom_filter_index_reader_num.get_value()); + + RuntimeProfile::Counter* index_page_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* index_page_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderNum", TUnit::UNIT); + COUNTER_SET(index_page_reader_mem_bytes_counter, g_index_page_reader_mem_bytes.get_value()); + COUNTER_SET(index_page_reader_num_counter, g_index_page_reader_num.get_value()); + + RuntimeProfile::Counter* indexed_column_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* indexed_column_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderNum", TUnit::UNIT); + COUNTER_SET(indexed_column_reader_mem_bytes_counter, + g_indexed_column_reader_mem_bytes.get_value()); + COUNTER_SET(indexed_column_reader_num_counter, g_indexed_column_reader_num.get_value()); + + RuntimeProfile::Counter* inverted_index_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* inverted_index_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderNum", TUnit::UNIT); + COUNTER_SET(inverted_index_reader_mem_bytes_counter, + g_inverted_index_reader_mem_bytes.get_value()); + COUNTER_SET(inverted_index_reader_num_counter, g_inverted_index_reader_num.get_value()); + + RuntimeProfile::Counter* ordinal_index_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* ordinal_index_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderNum", TUnit::UNIT); + COUNTER_SET(ordinal_index_reader_mem_bytes_counter, + g_ordinal_index_reader_mem_bytes.get_value()); + COUNTER_SET(ordinal_index_reader_num_counter, g_ordinal_index_reader_num.get_value()); + + RuntimeProfile::Counter* zone_map_index_reader_mem_bytes_counter = + ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderMemBytes", TUnit::BYTES); + RuntimeProfile::Counter* zone_map_index_reader_num_counter = + ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderNum", TUnit::UNIT); + COUNTER_SET(zone_map_index_reader_mem_bytes_counter, + g_zone_map_index_reader_mem_bytes.get_value()); + COUNTER_SET(zone_map_index_reader_num_counter, g_zone_map_index_reader_num.get_value()); +} + +}; // namespace doris diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp index 256f4d35313d13..ac3a2a7a1dc5c2 100644 --- a/be/src/olap/rowset/rowset.cpp +++ b/be/src/olap/rowset/rowset.cpp @@ -27,8 +27,6 @@ namespace doris { -static bvar::Adder g_total_rowset_num("doris_total_rowset_num"); - Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta, std::string tablet_path) : _rowset_meta(std::move(rowset_meta)), @@ -56,11 +54,6 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta, } // build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema _schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema; - g_total_rowset_num << 1; -} - -Rowset::~Rowset() { - g_total_rowset_num << -1; } Status Rowset::load(bool use_cache) { diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index e1a2347f6aeaa8..98d88ba19f2068 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -33,6 +33,7 @@ #include "common/logging.h" #include "common/status.h" +#include "olap/metadata_adder.h" #include "olap/olap_common.h" #include "olap/rowset/rowset_meta.h" #include "olap/tablet_schema.h" @@ -116,10 +117,8 @@ class RowsetStateMachine { RowsetState _rowset_state; }; -class Rowset : public std::enable_shared_from_this { +class Rowset : public std::enable_shared_from_this, public MetadataAdder { public: - virtual ~Rowset(); - // Open all segment files in this rowset and load necessary metadata. // - `use_cache` : whether to use fd cache, only applicable to alpha rowset now // diff --git a/be/src/runtime/process_profile.cpp b/be/src/runtime/process_profile.cpp index d91aedbeac2025..60b381e9c31578 100644 --- a/be/src/runtime/process_profile.cpp +++ b/be/src/runtime/process_profile.cpp @@ -19,6 +19,7 @@ #include +#include "olap/metadata_adder.h" #include "runtime/memory/memory_profile.h" namespace doris { @@ -37,8 +38,15 @@ void ProcessProfile::refresh_profile() { std::unique_ptr process_profile = std::make_unique("ProcessProfile"); _memory_profile->make_memory_profile(process_profile.get()); - _process_profile.set(std::move(process_profile)); // TODO make other profile + + // 3. dump object heap + RuntimeProfile* object_heap_dump_snapshot = + process_profile->create_child("ObjectHeapDump", true, false); + MetadataAdder::dump_metadata_object(object_heap_dump_snapshot); + // TODO dump other object (block, column, etc.) + + _process_profile.set(std::move(process_profile)); } } // namespace doris