Skip to content

Commit

Permalink
[pick](branch-3.0) pick #44647 (#45839)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xinyiZzz
xinyiZzz authored Dec 25, 2024
1 parent dda2469 commit 207327f
Show file tree
Hide file tree
Showing 4 changed files with 146 additions and 16 deletions.
140 changes: 135 additions & 5 deletions be/src/olap/metadata_adder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,16 @@
#include <bvar/bvar.h>
#include <stdint.h>

#include "util/runtime_profile.h"

namespace doris {

inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");

inline bvar::Adder<int64_t> g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");

inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");

Expand All @@ -37,8 +42,8 @@ inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");

inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
inline bvar::Adder<int64_t> g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");

inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
Expand Down Expand Up @@ -104,6 +109,8 @@ class MetadataAdder {
public:
MetadataAdder();

static void dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot);

protected:
MetadataAdder(const MetadataAdder& other);

Expand Down Expand Up @@ -159,6 +166,8 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
}
if constexpr (std::is_same_v<T, RowsetMeta>) {
g_rowset_meta_mem_bytes << val;
} else if constexpr (std::is_same_v<T, Rowset>) {
g_all_rowsets_mem_bytes << val;
} else if constexpr (std::is_same_v<T, TabletMeta>) {
g_tablet_meta_mem_bytes << val;
} else if constexpr (std::is_same_v<T, TabletColumn>) {
Expand All @@ -168,7 +177,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
} else if constexpr (std::is_same_v<T, TabletSchema>) {
g_tablet_schema_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
g_segment_mem_bytes << val;
g_all_segments_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
g_column_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
Expand All @@ -185,6 +194,9 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
g_ordinal_index_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
g_zone_map_index_reader_mem_bytes << val;
} else {
LOG(FATAL) << "add_mem_size not match class type: " << typeid(T).name() << ", " << val;
__builtin_unreachable();
}
}

Expand All @@ -195,6 +207,8 @@ void MetadataAdder<T>::add_num(int64_t val) {
}
if constexpr (std::is_same_v<T, RowsetMeta>) {
g_rowset_meta_num << val;
} else if constexpr (std::is_same_v<T, Rowset>) {
g_all_rowsets_num << val;
} else if constexpr (std::is_same_v<T, TabletMeta>) {
g_tablet_meta_num << val;
} else if constexpr (std::is_same_v<T, TabletColumn>) {
Expand All @@ -204,7 +218,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
} else if constexpr (std::is_same_v<T, TabletSchema>) {
g_tablet_schema_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
g_segment_num << val;
g_all_segments_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
g_column_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
Expand All @@ -221,7 +235,123 @@ void MetadataAdder<T>::add_num(int64_t val) {
g_ordinal_index_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
g_zone_map_index_reader_num << val;
} else {
LOG(FATAL) << "add_num not match class type: " << typeid(T).name() << ", " << val;
__builtin_unreachable();
}
}

}; // namespace doris
template <typename T>
void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot) {
RuntimeProfile::Counter* rowset_meta_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* rowset_meta_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaNum", TUnit::UNIT);
COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value());
COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());

RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* all_rowsets_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT);
COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value());
COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());

RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_meta_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaNum", TUnit::UNIT);
COUNTER_SET(tablet_meta_mem_bytes_counter, g_tablet_meta_mem_bytes.get_value());
COUNTER_SET(tablet_meta_num_counter, g_tablet_meta_num.get_value());

RuntimeProfile::Counter* tablet_column_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_column_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnNum", TUnit::UNIT);
COUNTER_SET(tablet_column_mem_bytes_counter, g_tablet_column_mem_bytes.get_value());
COUNTER_SET(tablet_column_num_counter, g_tablet_column_num.get_value());

RuntimeProfile::Counter* tablet_index_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_index_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexNum", TUnit::UNIT);
COUNTER_SET(tablet_index_mem_bytes_counter, g_tablet_index_mem_bytes.get_value());
COUNTER_SET(tablet_index_num_counter, g_tablet_index_num.get_value());

RuntimeProfile::Counter* tablet_schema_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_schema_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaNum", TUnit::UNIT);
COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value());
COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());

RuntimeProfile::Counter* all_segments_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* all_segments_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT);
COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value());
COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());

RuntimeProfile::Counter* column_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* column_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderNum", TUnit::UNIT);
COUNTER_SET(column_reader_mem_bytes_counter, g_column_reader_mem_bytes.get_value());
COUNTER_SET(column_reader_num_counter, g_column_reader_num.get_value());

RuntimeProfile::Counter* bitmap_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* bitmap_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderNum", TUnit::UNIT);
COUNTER_SET(bitmap_index_reader_mem_bytes_counter, g_bitmap_index_reader_mem_bytes.get_value());
COUNTER_SET(bitmap_index_reader_num_counter, g_bitmap_index_reader_num.get_value());

RuntimeProfile::Counter* bloom_filter_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* filter_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderNum", TUnit::UNIT);
COUNTER_SET(bloom_filter_index_reader_mem_bytes_counter,
g_bloom_filter_index_reader_mem_bytes.get_value());
COUNTER_SET(filter_index_reader_num_counter, g_bloom_filter_index_reader_num.get_value());

RuntimeProfile::Counter* index_page_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* index_page_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderNum", TUnit::UNIT);
COUNTER_SET(index_page_reader_mem_bytes_counter, g_index_page_reader_mem_bytes.get_value());
COUNTER_SET(index_page_reader_num_counter, g_index_page_reader_num.get_value());

RuntimeProfile::Counter* indexed_column_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* indexed_column_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderNum", TUnit::UNIT);
COUNTER_SET(indexed_column_reader_mem_bytes_counter,
g_indexed_column_reader_mem_bytes.get_value());
COUNTER_SET(indexed_column_reader_num_counter, g_indexed_column_reader_num.get_value());

RuntimeProfile::Counter* inverted_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* inverted_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderNum", TUnit::UNIT);
COUNTER_SET(inverted_index_reader_mem_bytes_counter,
g_inverted_index_reader_mem_bytes.get_value());
COUNTER_SET(inverted_index_reader_num_counter, g_inverted_index_reader_num.get_value());

RuntimeProfile::Counter* ordinal_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* ordinal_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderNum", TUnit::UNIT);
COUNTER_SET(ordinal_index_reader_mem_bytes_counter,
g_ordinal_index_reader_mem_bytes.get_value());
COUNTER_SET(ordinal_index_reader_num_counter, g_ordinal_index_reader_num.get_value());

RuntimeProfile::Counter* zone_map_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* zone_map_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderNum", TUnit::UNIT);
COUNTER_SET(zone_map_index_reader_mem_bytes_counter,
g_zone_map_index_reader_mem_bytes.get_value());
COUNTER_SET(zone_map_index_reader_num_counter, g_zone_map_index_reader_num.get_value());
}

}; // namespace doris
7 changes: 0 additions & 7 deletions be/src/olap/rowset/rowset.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@

namespace doris {

static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");

Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta,
std::string tablet_path)
: _rowset_meta(std::move(rowset_meta)),
Expand Down Expand Up @@ -56,11 +54,6 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta,
}
// build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
_schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema;
g_total_rowset_num << 1;
}

Rowset::~Rowset() {
g_total_rowset_num << -1;
}

Status Rowset::load(bool use_cache) {
Expand Down
5 changes: 2 additions & 3 deletions be/src/olap/rowset/rowset.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

#include "common/logging.h"
#include "common/status.h"
#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/tablet_schema.h"
Expand Down Expand Up @@ -116,10 +117,8 @@ class RowsetStateMachine {
RowsetState _rowset_state;
};

class Rowset : public std::enable_shared_from_this<Rowset> {
class Rowset : public std::enable_shared_from_this<Rowset>, public MetadataAdder<Rowset> {
public:
virtual ~Rowset();

// Open all segment files in this rowset and load necessary metadata.
// - `use_cache` : whether to use fd cache, only applicable to alpha rowset now
//
Expand Down
10 changes: 9 additions & 1 deletion be/src/runtime/process_profile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <memory>

#include "olap/metadata_adder.h"
#include "runtime/memory/memory_profile.h"

namespace doris {
Expand All @@ -37,8 +38,15 @@ void ProcessProfile::refresh_profile() {
std::unique_ptr<RuntimeProfile> process_profile =
std::make_unique<RuntimeProfile>("ProcessProfile");
_memory_profile->make_memory_profile(process_profile.get());
_process_profile.set(std::move(process_profile));
// TODO make other profile

// 3. dump object heap
RuntimeProfile* object_heap_dump_snapshot =
process_profile->create_child("ObjectHeapDump", true, false);
MetadataAdder<ProcessProfile>::dump_metadata_object(object_heap_dump_snapshot);
// TODO dump other object (block, column, etc.)

_process_profile.set(std::move(process_profile));
}

} // namespace doris

0 comments on commit 207327f

Please sign in to comment.