From cebf6481da4513e6ee1bb88f1d5c01fcea37e545 Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Tue, 10 Dec 2024 17:05:46 +0800 Subject: [PATCH] [improve](mow) merge and remove old version of delete bitmap when cumulative compaction is done on local mode (#41636) pr https://github.com/apache/doris/pull/40204 remove delete bitmap on cloud mode, this pr is used for local mode --- be/src/cloud/cloud_cumulative_compaction.cpp | 28 +--- .../action/delete_bitmap_action.cpp} | 36 ++--- .../action/delete_bitmap_action.h} | 13 +- be/src/olap/compaction.cpp | 64 +++++++++ be/src/olap/compaction.h | 8 ++ be/src/olap/rowset/beta_rowset_writer.cpp | 3 +- be/src/olap/tablet.cpp | 10 ++ be/src/olap/tablet_meta.cpp | 18 ++- be/src/service/http_service.cpp | 21 ++- ...on_remove_old_version_delete_bitmap.groovy | 134 +++++++++--------- 10 files changed, 212 insertions(+), 123 deletions(-) rename be/src/{cloud/cloud_delete_bitmap_action.cpp => http/action/delete_bitmap_action.cpp} (83%) rename be/src/{cloud/cloud_delete_bitmap_action.h => http/action/delete_bitmap_action.h} (79%) diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index f63054563aa18a..6b4dbe360da651 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -383,28 +383,10 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { } std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); if (!pre_rowsets.empty()) { - auto pre_max_version = _output_rowset->version().second; - DeleteBitmapPtr new_delete_bitmap = - std::make_shared(_tablet->tablet_meta()->tablet_id()); std::vector> to_remove_vec; - for (auto& rowset : pre_rowsets) { - if (rowset->rowset_meta()->total_disk_size() == 0) { - continue; - } - for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) { - rowset->rowset_id().to_string(); - DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0}; - DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version}; - auto d = _tablet->tablet_meta()->delete_bitmap().get_agg( - {rowset->rowset_id(), seg_id, pre_max_version}); - to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end)); - if (d->isEmpty()) { - continue; - } - new_delete_bitmap->set(end, *d); - } - } + DeleteBitmapPtr new_delete_bitmap = nullptr; + agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); if (!new_delete_bitmap->empty()) { // store agg delete bitmap DBUG_EXECUTE_IF("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed", @@ -424,9 +406,9 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { } _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), to_remove_vec); - DBUG_EXECUTE_IF( - "CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", - { static_cast(_tablet.get())->delete_expired_stale_rowsets(); }); + DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { + static_cast(_tablet.get())->delete_expired_stale_rowsets(); + }); } } return Status::OK(); diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/http/action/delete_bitmap_action.cpp similarity index 83% rename from be/src/cloud/cloud_delete_bitmap_action.cpp rename to be/src/http/action/delete_bitmap_action.cpp index 3d834bfe7b373c..2fa0a73c2f338c 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.cpp +++ b/be/src/http/action/delete_bitmap_action.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "cloud_delete_bitmap_action.h" +#include "delete_bitmap_action.h" #include #include @@ -34,8 +34,10 @@ #include #include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" #include "common/logging.h" #include "common/status.h" #include "gutil/strings/substitute.h" @@ -44,7 +46,6 @@ #include "http/http_request.h" #include "http/http_status.h" #include "olap/olap_define.h" -#include "olap/storage_engine.h" #include "olap/tablet_manager.h" #include "util/doris_metrics.h" #include "util/stopwatch.hpp" @@ -59,10 +60,9 @@ constexpr std::string_view HEADER_JSON = "application/json"; } // namespace -CloudDeleteBitmapAction::CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, - CloudStorageEngine& engine, - TPrivilegeHier::type hier, - TPrivilegeType::type ptype) +DeleteBitmapAction::DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, + BaseStorageEngine& engine, TPrivilegeHier::type hier, + TPrivilegeType::type ptype) : HttpHandlerWithAuth(exec_env, hier, ptype), _engine(engine), _delete_bitmap_action_type(ctype) {} @@ -80,8 +80,8 @@ static Status _check_param(HttpRequest* req, uint64_t* tablet_id) { return Status::OK(); } -Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req, - std::string* json_result) { +Status DeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req, + std::string* json_result) { uint64_t tablet_id = 0; // check & retrieve tablet_id from req if it contains RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed"); @@ -89,11 +89,15 @@ Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpReque return Status::InternalError("check param failed: missing tablet_id"); } - CloudTabletSPtr tablet = DORIS_TRY(_engine.tablet_mgr().get_tablet(tablet_id)); + BaseTabletSPtr tablet = nullptr; + if (config::is_cloud_mode()) { + tablet = DORIS_TRY(_engine.to_cloud().tablet_mgr().get_tablet(tablet_id)); + } else { + tablet = _engine.to_local().tablet_manager()->get_tablet(tablet_id); + } if (tablet == nullptr) { return Status::NotFound("Tablet not found. tablet_id={}", tablet_id); } - auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality(); auto size = tablet->tablet_meta()->delete_bitmap().get_size(); @@ -115,8 +119,8 @@ Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpReque return Status::OK(); } -Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, - std::string* json_result) { +Status DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, + std::string* json_result) { uint64_t tablet_id = 0; // check & retrieve tablet_id from req if it contains RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed"); @@ -124,14 +128,14 @@ Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* return Status::InternalError("check param failed: missing tablet_id"); } TabletMetaSharedPtr tablet_meta; - auto st = _engine.meta_mgr().get_tablet_meta(tablet_id, &tablet_meta); + auto st = _engine.to_cloud().meta_mgr().get_tablet_meta(tablet_id, &tablet_meta); if (!st.ok()) { LOG(WARNING) << "failed to get_tablet_meta tablet=" << tablet_id << ", st=" << st.to_string(); return st; } - auto tablet = std::make_shared(_engine, std::move(tablet_meta)); - st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true); + auto tablet = std::make_shared(_engine.to_cloud(), std::move(tablet_meta)); + st = _engine.to_cloud().meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true); if (!st.ok()) { LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st; return st; @@ -157,7 +161,7 @@ Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* return Status::OK(); } -void CloudDeleteBitmapAction::handle(HttpRequest* req) { +void DeleteBitmapAction::handle(HttpRequest* req) { req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data()); if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) { std::string json_result; diff --git a/be/src/cloud/cloud_delete_bitmap_action.h b/be/src/http/action/delete_bitmap_action.h similarity index 79% rename from be/src/cloud/cloud_delete_bitmap_action.h rename to be/src/http/action/delete_bitmap_action.h index ce507ee9991757..284e8dbcf5705b 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.h +++ b/be/src/http/action/delete_bitmap_action.h @@ -21,9 +21,9 @@ #include -#include "cloud/cloud_storage_engine.h" #include "common/status.h" #include "http/http_handler_with_auth.h" +#include "olap/storage_engine.h" #include "olap/tablet.h" namespace doris { @@ -35,13 +35,12 @@ class ExecEnv; enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2 }; /// This action is used for viewing the delete bitmap status -class CloudDeleteBitmapAction : public HttpHandlerWithAuth { +class DeleteBitmapAction : public HttpHandlerWithAuth { public: - CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, - CloudStorageEngine& engine, TPrivilegeHier::type hier, - TPrivilegeType::type ptype); + DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, BaseStorageEngine& engine, + TPrivilegeHier::type hier, TPrivilegeType::type ptype); - ~CloudDeleteBitmapAction() override = default; + ~DeleteBitmapAction() override = default; void handle(HttpRequest* req) override; @@ -50,7 +49,7 @@ class CloudDeleteBitmapAction : public HttpHandlerWithAuth { Status _handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string* json_result); private: - CloudStorageEngine& _engine; + BaseStorageEngine& _engine; DeleteBitmapActionType _delete_bitmap_action_type; }; #include "common/compile_check_end.h" diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 8c45c20f799427..7b2bb5d4ee3cfb 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -929,6 +929,33 @@ Status CloudCompactionMixin::update_delete_bitmap() { return Status::OK(); } +void Compaction::agg_and_remove_old_version_delete_bitmap( + std::vector& pre_rowsets, + std::vector>& + to_remove_vec, + DeleteBitmapPtr& new_delete_bitmap) { + // agg previously rowset old version delete bitmap + auto pre_max_version = _output_rowset->version().second; + new_delete_bitmap = std::make_shared(_tablet->tablet_meta()->tablet_id()); + for (auto& rowset : pre_rowsets) { + if (rowset->rowset_meta()->total_disk_size() == 0) { + continue; + } + for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) { + rowset->rowset_id().to_string(); + DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0}; + DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version}; + auto d = _tablet->tablet_meta()->delete_bitmap().get_agg( + {rowset->rowset_id(), seg_id, pre_max_version}); + to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end)); + if (d->isEmpty()) { + continue; + } + new_delete_bitmap->set(end, *d); + } + } +} + Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) { // only do index compaction for dup_keys and unique_keys with mow enabled if (config::inverted_index_compaction_enable && @@ -1103,6 +1130,13 @@ Status CompactionMixin::modify_rowsets() { tablet()->delete_expired_stale_rowset(); } + if (config::enable_delete_bitmap_merge_on_compaction && + compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION && + _tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { + process_old_version_delete_bitmap(); + } + int64_t cur_max_version = 0; { std::shared_lock rlock(_tablet->get_header_lock()); @@ -1121,6 +1155,36 @@ Status CompactionMixin::modify_rowsets() { return Status::OK(); } +void CompactionMixin::process_old_version_delete_bitmap() { + std::vector pre_rowsets {}; + for (const auto& it : tablet()->rowset_map()) { + if (it.first.second < _input_rowsets.front()->start_version()) { + pre_rowsets.emplace_back(it.second); + } + } + std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); + if (!pre_rowsets.empty()) { + std::vector> + to_remove_vec; + DeleteBitmapPtr new_delete_bitmap = nullptr; + agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); + if (!new_delete_bitmap->empty()) { + // store agg delete bitmap + Version version(_input_rowsets.front()->start_version(), + _input_rowsets.back()->end_version()); + for (auto it = new_delete_bitmap->delete_bitmap.begin(); + it != new_delete_bitmap->delete_bitmap.end(); it++) { + _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); + } + _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), + to_remove_vec); + DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { + static_cast(_tablet.get())->delete_expired_stale_rowset(); + }); + } + } +} + bool CompactionMixin::_check_if_includes_input_rowsets( const RowsetIdUnorderedSet& commit_rowset_ids_set) const { std::vector commit_rowset_ids {}; diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index 7f92a6c5f4d7dc..ccabf7dadb4733 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -86,6 +86,12 @@ class Compaction { virtual Status update_delete_bitmap() = 0; + void agg_and_remove_old_version_delete_bitmap( + std::vector& pre_rowsets, + std::vector>& + to_remove_vec, + DeleteBitmapPtr& new_delete_bitmap); + // the root tracker for this compaction std::shared_ptr _mem_tracker; @@ -162,6 +168,8 @@ class CompactionMixin : public Compaction { Status do_compact_ordered_rowsets(); + void process_old_version_delete_bitmap(); + bool _check_if_includes_input_rowsets(const RowsetIdUnorderedSet& commit_rowset_ids_set) const; PendingRowsetGuard _pending_rs_guard; diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index ab5bc48db80b00..dc155efe0165bc 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -337,7 +337,8 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " << _context.tablet->tablet_id() << ", rowset_ids: " << _context.mow_context->rowset_ids.size() << ", cur max_version: " << _context.mow_context->max_version - << ", transaction_id: " << _context.mow_context->txn_id + << ", transaction_id: " << _context.mow_context->txn_id << ", delete_bitmap_count: " + << _context.tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count() << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows; return Status::OK(); } diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 379fb6eec3cb43..644ca9133eb885 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -817,10 +817,13 @@ void Tablet::delete_expired_stale_rowset() { auto old_meta_size = _tablet_meta->all_stale_rs_metas().size(); // do delete operation + std::vector version_to_delete; auto to_delete_iter = stale_version_path_map.begin(); while (to_delete_iter != stale_version_path_map.end()) { std::vector& to_delete_version = to_delete_iter->second->timestamped_versions(); + int64_t start_version = -1; + int64_t end_version = -1; for (auto& timestampedVersion : to_delete_version) { auto it = _stale_rs_version_map.find(timestampedVersion->version()); if (it != _stale_rs_version_map.end()) { @@ -841,10 +844,17 @@ void Tablet::delete_expired_stale_rowset() { << timestampedVersion->version().second << "] not find in stale rs version map"; } + if (start_version < 0) { + start_version = timestampedVersion->version().first; + } + end_version = timestampedVersion->version().second; _delete_stale_rowset_by_version(timestampedVersion->version()); } + Version version(start_version, end_version); + version_to_delete.emplace_back(version.to_string()); to_delete_iter++; } + _tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete); bool reconstructed = _reconstruct_version_tracker_if_necessary(); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 3247f34656fb5d..43b0d5d8bd0ae0 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1091,7 +1091,9 @@ uint64_t DeleteBitmap::cardinality() const { std::shared_lock l(lock); uint64_t res = 0; for (auto entry : delete_bitmap) { - res += entry.second.cardinality(); + if (std::get<1>(entry.first) != DeleteBitmap::INVALID_SEGMENT_ID) { + res += entry.second.cardinality(); + } } return res; } @@ -1100,7 +1102,9 @@ uint64_t DeleteBitmap::get_size() const { std::shared_lock l(lock); uint64_t charge = 0; for (auto& [k, v] : delete_bitmap) { - charge += v.getSizeInBytes(); + if (std::get<1>(k) != DeleteBitmap::INVALID_SEGMENT_ID) { + charge += v.getSizeInBytes(); + } } return charge; } @@ -1219,7 +1223,7 @@ void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vectorstorage_engine().to_cloud(); @@ -1232,7 +1236,13 @@ void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector(it->first) != DeleteBitmap::INVALID_SEGMENT_ID) { + count++; + } + } + return count; } // We cannot just copy the underlying memory to construct a string diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index c4b01a641987e6..eb0824170b30cc 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -25,7 +25,6 @@ #include #include "cloud/cloud_compaction_action.h" -#include "cloud/cloud_delete_bitmap_action.h" #include "cloud/config.h" #include "cloud/injection_point_action.h" #include "common/config.h" @@ -43,6 +42,7 @@ #include "http/action/compaction_score_action.h" #include "http/action/config_action.h" #include "http/action/debug_point_action.h" +#include "http/action/delete_bitmap_action.h" #include "http/action/download_action.h" #include "http/action/download_binlog_action.h" #include "http/action/file_cache_action.h" @@ -389,6 +389,13 @@ void HttpService::register_local_handler(StorageEngine& engine) { _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status", run_status_compaction_action); + + DeleteBitmapAction* count_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local", + count_delete_bitmap_action); + CheckTabletSegmentAction* check_tablet_segment_action = _pool.add(new CheckTabletSegmentAction( _env, engine, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::POST, "/api/check_tablet_segment_lost", @@ -437,14 +444,14 @@ void HttpService::register_cloud_handler(CloudStorageEngine& engine) { TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status", run_status_compaction_action); - CloudDeleteBitmapAction* count_local_delete_bitmap_action = - _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, - TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + DeleteBitmapAction* count_local_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local", count_local_delete_bitmap_action); - CloudDeleteBitmapAction* count_ms_delete_bitmap_action = - _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine, - TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + DeleteBitmapAction* count_ms_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_ms", count_ms_delete_bitmap_action); #ifdef ENABLE_INJECTION_POINT diff --git a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy index a36cb4579ca487..e14bdbbb2c6505 100644 --- a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy @@ -18,9 +18,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { - if (!isCloudMode()) { - return - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] def backendId_to_params = [string: [:]] @@ -163,7 +160,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { def timeout = 10000 sql """ DROP TABLE IF EXISTS ${testTable}""" def testTableDDL = """ - create table ${testTable} + create table ${testTable} ( `plan_id` bigint(20) NOT NULL, `target_id` int(20) NOT NULL, @@ -189,7 +186,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { set_be_param("tablet_rowset_stale_sweep_time_sec", "0") try { - GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") + GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") // 1. test normal sql "sync" sql """ INSERT INTO ${testTable} VALUES (0,0,'1'),(1,1,'1'); """ @@ -219,18 +216,20 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { // before compaction, delete_bitmap_count is (rowsets num - 1) local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 7) - assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 7) assertTrue(local_delete_bitmap_cardinality == 7) - assertTrue(local_delete_bitmap_cardinality == ms_delete_bitmap_cardinality) + + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 7) + assertTrue(ms_delete_bitmap_cardinality == 7) + } assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], @@ -265,18 +264,20 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { // before compaction, local delete_bitmap_count is (total rowsets num - 1), ms delete_bitmap_count is new rowset num String trigger_backend_id = tablet.BackendId local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 12) - assertTrue(ms_delete_bitmap_count == 5) - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 12) assertTrue(local_delete_bitmap_cardinality == 17) - assertTrue(ms_delete_bitmap_cardinality == 10) + + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 5) + assertTrue(ms_delete_bitmap_cardinality == 10) + } getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], @@ -285,20 +286,21 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); Thread.sleep(1000) - // after compaction, delete_bitmap_count is 1 + // after compaction, delete_bitmap_count is 1, cardinality is 2, check it local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 1) - assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 1) assertTrue(local_delete_bitmap_cardinality == 2) - assertTrue(ms_delete_bitmap_cardinality == 2) + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 1) + assertTrue(ms_delete_bitmap_cardinality == 2) + } } qt_sql "select * from ${testTable} order by plan_id" @@ -319,37 +321,39 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { qt_sql "select * from ${testTable} order by plan_id" GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed") - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - String trigger_backend_id = tablet.BackendId - - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 6) - assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) - - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_cardinality == 12) - assertTrue(ms_delete_bitmap_cardinality == 12) - - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - - // update fail, local delete_bitmap_count will not change - Thread.sleep(1000) - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 6) + if (isCloudMode()) { + for (def tablet in tablets) { + String tablet_id = tablet.TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + String trigger_backend_id = tablet.BackendId + + local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + assertTrue(local_delete_bitmap_count == 6) + assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) + + local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_cardinality == 12) + assertTrue(ms_delete_bitmap_cardinality == 12) + + getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); + assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], + "cumulative", tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) + getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); + + // update fail, local delete_bitmap_count will not change + Thread.sleep(1000) + local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) + assertTrue(local_delete_bitmap_count == 6) + } } qt_sql "select * from ${testTable} order by plan_id" @@ -372,7 +376,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { } finally { reset_be_param("compaction_promotion_version_count") reset_be_param("tablet_rowset_stale_sweep_time_sec") - GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") + GetDebugPoint().disableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed") }