From 624dd8091468e1519e53f4e598bd2e429bb60399 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Thu, 21 Nov 2024 00:41:04 +0800 Subject: [PATCH] 3 --- cloud/src/meta-service/meta_service.cpp | 56 ++++++++++- .../meta_service_tablet_stats.cpp | 95 +++++++++++++++++++ .../meta-service/meta_service_tablet_stats.h | 8 ++ 3 files changed, 155 insertions(+), 4 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index b398de8a4345b1c..a8910b10e006cde 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1650,13 +1651,60 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); - msg = fmt::format("failed to create txn, tablet_id={}", idx.tablet_id()); + msg = fmt::format("batch get tablet stats failed to create txn"); return; } - for (auto& i : request->tablet_idx()) { + + // batch get tablet idx + DCHECK_LE(request->tablet_idx().size(), 1000); + std::vector tablet_idx_keys; + std::string key; + for (const auto& i : request->tablet_idx()) { + meta_tablet_idx_key({instance_id, i.tablet_id()}, &key); + tablet_idx_keys.emplace_back(key); + } + std::vector tablet_idx_pb_vec; + auto st = batch_get_tablet_idx(txn.get(), tablet_idx_keys, tablet_idx_pb_vec); + if (st.code() != MetaServiceCode::OK) { + LOG_WARNING("failed to batch get tablet idx").tag("err", st.msg()); + return; + } + + // batch get tablet stats + std::vector tablet_stats_keys; + for (const auto& idx : tablet_idx_pb_vec) { + stats_tablet_key({instance_id, idx.table_id(), idx.index_id(), idx.partition_id(), + idx.tablet_id()}, + &key); + tablet_stats_keys.emplace_back(key); + stats_tablet_data_size_key({instance_id, idx.table_id(), idx.index_id(), + idx.partition_id(), idx.tablet_id()}, + &key); + tablet_stats_keys.emplace_back(key); + stats_tablet_num_rows_key({instance_id, idx.table_id(), idx.index_id(), + idx.partition_id(), idx.tablet_id()}, + &key); + tablet_stats_keys.emplace_back(key); + stats_tablet_num_rowsets_key({instance_id, idx.table_id(), idx.index_id(), + idx.partition_id(), idx.tablet_id()}, + &key); + tablet_stats_keys.emplace_back(key); + stats_tablet_num_segs_key({instance_id, idx.table_id(), idx.index_id(), + idx.partition_id(), idx.tablet_id()}, + &key); + tablet_stats_keys.emplace_back(key); + } + std::vector tablet_stats_pb_vec; + st = batch_get_tablet_stats(txn.get(), tablet_stats_keys, tablet_stats_pb_vec); + if (st.code() != MetaServiceCode::OK) { + LOG_WARNING("failed to batch get tablet stats").tag("err", st.msg()); + response->clear_tablet_stats(); + return; + } + for (const auto& stat : tablet_stats_pb_vec) { + auto* stat_ptr = response->add_tablet_stats(); + stat_ptr->CopyFrom(stat); } - get_batch_tablet_idx(); - internal_get_batch_tablet_stats(); } } diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp b/cloud/src/meta-service/meta_service_tablet_stats.cpp index cecccbd67673ad1..7ad73c08ef4dbea 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.cpp +++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "common/logging.h" #include "common/util.h" @@ -165,6 +166,100 @@ void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transact merge_tablet_stats(stats, detached_stats); } +MetaServiceResponseStatus batch_get_tablet_idx(Transaction* txn, + const std::vector tablet_idx_keys, + std::vector& tablet_idx_pb_vec) { + MetaServiceResponseStatus st; + st.set_code(MetaServiceCode::OK); + std::vector> tablet_idx_values; + auto err = txn->batch_get(&tablet_idx_values, tablet_idx_keys); + if (err != TxnErrorCode::TXN_OK) { + st.set_code(cast_as(err)); + } + for (const auto& idx : tablet_idx_values) { + if (idx.has_value()) { + TabletIndexPB tablet_idx_pb; + tablet_idx_pb.ParseFromArray(idx.value().data(), idx.value().size()); + tablet_idx_pb_vec.emplace_back(std::move(tablet_idx_pb)); + } else { + LOG_WARNING("failed to batch get tablet idx, miss value"); + st.set_msg("failed to batch get tablet idx, miss value"); + st.set_code(cast_as(TxnErrorCode::TXN_KEY_NOT_FOUND)); + } + } + return st; +} + +MetaServiceResponseStatus batch_get_tablet_stats(Transaction* txn, + const std::vector tablet_stats_keys, + std::vector& tablet_stats_pb_vec) { + MetaServiceResponseStatus st; + st.set_code(MetaServiceCode::OK); + std::vector> tablet_stats_values; + auto err = txn->batch_get(&tablet_stats_values, tablet_stats_keys); + if (err != TxnErrorCode::TXN_OK) { + st.set_code(cast_as(err)); + } + + for (int i = 0; i < tablet_stats_values.size(); ++i) { + // aggregate + data_size + num_rows + num_rowsets + num_segments + TabletStatsPB tablet_stat_pb; + TabletStats detached_tablet_stat; + for (int j = 0; j < 5; ++j) { + auto stat = tablet_stats_values[i + j]; + int64_t val = 0; + bool value_miss = false; + // decode detached stat + if (stat.has_value() && j != 0) { + if (stat.value().size() != sizeof(val)) [[unlikely]] { + LOG(WARNING) << "malformed tablet stats data size value v.size=" + << stat.value().size(); + } + std::memcpy(&val, stat.value().data(), sizeof(val)); + if constexpr (std::endian::native == std::endian::big) { + val = bswap_64(val); + } + } else if (j != 0) { + value_miss = true; + } + + switch (j) { + // handle aggregate stat + case 0: + if (stat.has_value()) { + tablet_stat_pb.ParseFromArray(stat.value().data(), stat.value().size()); + } else { + LOG_WARNING("failed to batch get tablet stat, miss aggregate value"); + st.set_msg("failed to batch get tablet stat, miss aggregate value"); + st.set_code(cast_as(TxnErrorCode::TXN_KEY_NOT_FOUND)); + return st; + } + // handle data_size stat + case 1: + detached_tablet_stat.data_size = val; + // handle num_rows stat + case 2: + detached_tablet_stat.num_rows = val; + // handle num_rowsets stat + case 3: + detached_tablet_stat.num_rowsets = val; + // handle num_segs stat + case 4: + detached_tablet_stat.num_segs = val; + default: + return st; + } + if (value_miss) { + DCHECK_EQ(tablet_stat_pb.num_segments(), 0); + } + } + merge_tablet_stats(tablet_stat_pb, detached_tablet_stat); + tablet_stats_pb_vec.emplace_back(std::move(tablet_stat_pb)); + i += 5; + } + return st; +} + MetaServiceResponseStatus parse_fix_tablet_stats_param( std::shared_ptr resource_mgr, const std::string& table_id_str, const std::string& cloud_unique_id_str, int64_t& table_id, std::string& instance_id) { diff --git a/cloud/src/meta-service/meta_service_tablet_stats.h b/cloud/src/meta-service/meta_service_tablet_stats.h index a7aea5885a8e1a7..b511b1640c277da 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.h +++ b/cloud/src/meta-service/meta_service_tablet_stats.h @@ -49,6 +49,14 @@ void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transact const std::string& instance_id, const TabletIndexPB& idx, TabletStatsPB& stats, bool snapshot = false); +MetaServiceResponseStatus batch_get_tablet_idx(Transaction* txn, + const std::vector tablet_idx_keys, + std::vector& tablet_idx_pb_vec); + +MetaServiceResponseStatus batch_get_tablet_stats(Transaction* txn, + const std::vector tablet_stats_keys, + std::vector& tablet_stats_pb_vec); + // clang-format off /** * Get detached tablet stats via with given stats_kvs