Skip to content

Commit

Permalink
3
Browse files Browse the repository at this point in the history
  • Loading branch information
@Yukang-Lian
Yukang-Lian committed Nov 20, 2024
1 parent e9a454c commit 624dd80
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 4 deletions.
56 changes: 52 additions & 4 deletions cloud/src/meta-service/meta_service.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <fmt/core.h>
#include <gen_cpp/cloud.pb.h>
#include <gen_cpp/olap_file.pb.h>
#include <glog/logging.h>
#include <google/protobuf/util/json_util.h>
#include <rapidjson/prettywriter.h>
#include <rapidjson/schema.h>
Expand Down Expand Up @@ -1650,13 +1651,60 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro
TxnErrorCode err = txn_kv_->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::CREATE>(err);
msg = fmt::format("failed to create txn, tablet_id={}", idx.tablet_id());
msg = fmt::format("batch get tablet stats failed to create txn");
return;
}
for (auto& i : request->tablet_idx()) {

// batch get tablet idx
DCHECK_LE(request->tablet_idx().size(), 1000);
std::vector<std::string> tablet_idx_keys;
std::string key;
for (const auto& i : request->tablet_idx()) {
meta_tablet_idx_key({instance_id, i.tablet_id()}, &key);
tablet_idx_keys.emplace_back(key);
}
std::vector<TabletIndexPB> tablet_idx_pb_vec;
auto st = batch_get_tablet_idx(txn.get(), tablet_idx_keys, tablet_idx_pb_vec);
if (st.code() != MetaServiceCode::OK) {
LOG_WARNING("failed to batch get tablet idx").tag("err", st.msg());
return;
}

// batch get tablet stats
std::vector<std::string> tablet_stats_keys;
for (const auto& idx : tablet_idx_pb_vec) {
stats_tablet_key({instance_id, idx.table_id(), idx.index_id(), idx.partition_id(),
idx.tablet_id()},
&key);
tablet_stats_keys.emplace_back(key);
stats_tablet_data_size_key({instance_id, idx.table_id(), idx.index_id(),
idx.partition_id(), idx.tablet_id()},
&key);
tablet_stats_keys.emplace_back(key);
stats_tablet_num_rows_key({instance_id, idx.table_id(), idx.index_id(),
idx.partition_id(), idx.tablet_id()},
&key);
tablet_stats_keys.emplace_back(key);
stats_tablet_num_rowsets_key({instance_id, idx.table_id(), idx.index_id(),
idx.partition_id(), idx.tablet_id()},
&key);
tablet_stats_keys.emplace_back(key);
stats_tablet_num_segs_key({instance_id, idx.table_id(), idx.index_id(),
idx.partition_id(), idx.tablet_id()},
&key);
tablet_stats_keys.emplace_back(key);
}
std::vector<TabletStatsPB> tablet_stats_pb_vec;
st = batch_get_tablet_stats(txn.get(), tablet_stats_keys, tablet_stats_pb_vec);
if (st.code() != MetaServiceCode::OK) {
LOG_WARNING("failed to batch get tablet stats").tag("err", st.msg());
response->clear_tablet_stats();
return;
}
for (const auto& stat : tablet_stats_pb_vec) {
auto* stat_ptr = response->add_tablet_stats();
stat_ptr->CopyFrom(stat);
}
get_batch_tablet_idx();
internal_get_batch_tablet_stats();
}
}

Expand Down
95 changes: 95 additions & 0 deletions cloud/src/meta-service/meta_service_tablet_stats.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <memory>
#include <string>
#include <string_view>
#include <vector>

#include "common/logging.h"
#include "common/util.h"
Expand Down Expand Up @@ -165,6 +166,100 @@ void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transact
merge_tablet_stats(stats, detached_stats);
}

MetaServiceResponseStatus batch_get_tablet_idx(Transaction* txn,
const std::vector<std::string> tablet_idx_keys,
std::vector<TabletIndexPB>& tablet_idx_pb_vec) {
MetaServiceResponseStatus st;
st.set_code(MetaServiceCode::OK);
std::vector<std::optional<std::string>> tablet_idx_values;
auto err = txn->batch_get(&tablet_idx_values, tablet_idx_keys);
if (err != TxnErrorCode::TXN_OK) {
st.set_code(cast_as<ErrCategory::READ>(err));
}
for (const auto& idx : tablet_idx_values) {
if (idx.has_value()) {
TabletIndexPB tablet_idx_pb;
tablet_idx_pb.ParseFromArray(idx.value().data(), idx.value().size());
tablet_idx_pb_vec.emplace_back(std::move(tablet_idx_pb));
} else {
LOG_WARNING("failed to batch get tablet idx, miss value");
st.set_msg("failed to batch get tablet idx, miss value");
st.set_code(cast_as<ErrCategory::READ>(TxnErrorCode::TXN_KEY_NOT_FOUND));
}
}
return st;
}

MetaServiceResponseStatus batch_get_tablet_stats(Transaction* txn,
const std::vector<std::string> tablet_stats_keys,
std::vector<TabletStatsPB>& tablet_stats_pb_vec) {
MetaServiceResponseStatus st;
st.set_code(MetaServiceCode::OK);
std::vector<std::optional<std::string>> tablet_stats_values;
auto err = txn->batch_get(&tablet_stats_values, tablet_stats_keys);
if (err != TxnErrorCode::TXN_OK) {
st.set_code(cast_as<ErrCategory::READ>(err));
}

for (int i = 0; i < tablet_stats_values.size(); ++i) {
// aggregate + data_size + num_rows + num_rowsets + num_segments
TabletStatsPB tablet_stat_pb;
TabletStats detached_tablet_stat;
for (int j = 0; j < 5; ++j) {
auto stat = tablet_stats_values[i + j];
int64_t val = 0;
bool value_miss = false;
// decode detached stat
if (stat.has_value() && j != 0) {
if (stat.value().size() != sizeof(val)) [[unlikely]] {
LOG(WARNING) << "malformed tablet stats data size value v.size="
<< stat.value().size();
}
std::memcpy(&val, stat.value().data(), sizeof(val));
if constexpr (std::endian::native == std::endian::big) {
val = bswap_64(val);
}
} else if (j != 0) {
value_miss = true;
}

switch (j) {
// handle aggregate stat
case 0:
if (stat.has_value()) {
tablet_stat_pb.ParseFromArray(stat.value().data(), stat.value().size());
} else {
LOG_WARNING("failed to batch get tablet stat, miss aggregate value");
st.set_msg("failed to batch get tablet stat, miss aggregate value");
st.set_code(cast_as<ErrCategory::READ>(TxnErrorCode::TXN_KEY_NOT_FOUND));
return st;
}
// handle data_size stat
case 1:
detached_tablet_stat.data_size = val;
// handle num_rows stat
case 2:
detached_tablet_stat.num_rows = val;
// handle num_rowsets stat
case 3:
detached_tablet_stat.num_rowsets = val;
// handle num_segs stat
case 4:
detached_tablet_stat.num_segs = val;
default:
return st;
}
if (value_miss) {
DCHECK_EQ(tablet_stat_pb.num_segments(), 0);
}
}
merge_tablet_stats(tablet_stat_pb, detached_tablet_stat);
tablet_stats_pb_vec.emplace_back(std::move(tablet_stat_pb));
i += 5;
}
return st;
}

MetaServiceResponseStatus parse_fix_tablet_stats_param(
std::shared_ptr<ResourceManager> resource_mgr, const std::string& table_id_str,
const std::string& cloud_unique_id_str, int64_t& table_id, std::string& instance_id) {
Expand Down
8 changes: 8 additions & 0 deletions cloud/src/meta-service/meta_service_tablet_stats.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,14 @@ void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transact
const std::string& instance_id, const TabletIndexPB& idx,
TabletStatsPB& stats, bool snapshot = false);

MetaServiceResponseStatus batch_get_tablet_idx(Transaction* txn,
const std::vector<std::string> tablet_idx_keys,
std::vector<TabletIndexPB>& tablet_idx_pb_vec);

MetaServiceResponseStatus batch_get_tablet_stats(Transaction* txn,
const std::vector<std::string> tablet_stats_keys,
std::vector<TabletStatsPB>& tablet_stats_pb_vec);

// clang-format off
/**
* Get detached tablet stats via with given stats_kvs
Expand Down

0 comments on commit 624dd80

Please sign in to comment.