From 3e0cb52ad1fbce7aaafa81e332d90305e1ae3a3b Mon Sep 17 00:00:00 2001 From: Luwei Date: Fri, 22 Nov 2024 20:03:30 +0800 Subject: [PATCH] [improvement](compaction) reduce tablet skip compaction time (#44273) The time for tablet skip compaction is 120 seconds, which is too long. In the scenario of high-frequency import (mow), it leads to a high compaction score. Therefore, reducing the skip time to 10 seconds is necessary. --- be/src/common/config.cpp | 4 +++- be/src/common/config.h | 1 + be/src/olap/tablet.cpp | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index e376ab03bfdfe8..4931b61940d08b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -424,6 +424,8 @@ DEFINE_mDouble(base_compaction_min_data_ratio, "0.3"); DEFINE_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024"); DEFINE_Bool(enable_skip_tablet_compaction, "true"); +DEFINE_mInt32(skip_tablet_compaction_second, "10"); + // output rowset of cumulative compaction total disk size exceed this config size, // this rowset will be given to base compaction, unit is m byte. DEFINE_mInt64(compaction_promotion_size_mbytes, "1024"); @@ -459,7 +461,7 @@ DEFINE_mInt32(multi_get_max_threads, "10"); DEFINE_mInt64(total_permits_for_compaction_score, "10000"); // sleep interval in ms after generated compaction tasks -DEFINE_mInt32(generate_compaction_tasks_interval_ms, "10"); +DEFINE_mInt32(generate_compaction_tasks_interval_ms, "100"); // sleep interval in second after update replica infos DEFINE_mInt32(update_replica_infos_interval_seconds, "60"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 06f356f5fe8c38..15e384e7dfbd9d 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -478,6 +478,7 @@ DECLARE_mDouble(base_compaction_min_data_ratio); DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes); DECLARE_Bool(enable_skip_tablet_compaction); +DECLARE_mInt32(skip_tablet_compaction_second); // output rowset of cumulative compaction total disk size exceed this config size, // this rowset will be given to base compaction, unit is m byte. DECLARE_mInt64(compaction_promotion_size_mbytes); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index ad8550709a0283..5d20ba13c392b9 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -3860,10 +3860,10 @@ void Tablet::set_skip_compaction(bool skip, CompactionType compaction_type, int6 bool Tablet::should_skip_compaction(CompactionType compaction_type, int64_t now) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION && _skip_cumu_compaction && - now < _skip_cumu_compaction_ts + 120) { + now < _skip_cumu_compaction_ts + config::skip_tablet_compaction_second) { return true; } else if (compaction_type == CompactionType::BASE_COMPACTION && _skip_base_compaction && - now < _skip_base_compaction_ts + 120) { + now < _skip_base_compaction_ts + config::skip_tablet_compaction_second) { return true; } return false;