From ce8afba1d4e110903afe36bba6261d02326f77db Mon Sep 17 00:00:00 2001 From: zhangqiang Date: Mon, 23 Sep 2024 23:56:52 +0800 Subject: [PATCH 1/2] [BugFix] fix pk concurrent apply issue (#51225) Signed-off-by: sevev (cherry picked from commit 9dba8da03da06e23a29ab29ad6f1a903936caab5) # Conflicts: # be/src/storage/base_tablet.h --- be/src/storage/base_tablet.h | 6 ++++++ be/src/storage/tablet.h | 1 - be/src/storage/tablet_manager.cpp | 10 ++++++++++ be/src/storage/tablet_updates.h | 1 + be/test/storage/tablet_updates_test.cpp | 7 +++++++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/be/src/storage/base_tablet.h b/be/src/storage/base_tablet.h index 3164fe70c318a..d8321f1b9d3e8 100644 --- a/be/src/storage/base_tablet.h +++ b/be/src/storage/base_tablet.h @@ -115,9 +115,15 @@ class BaseTablet : public std::enable_shared_from_this { return flag; } +<<<<<<< HEAD protected: +======= + virtual size_t num_rows() const = 0; + +>>>>>>> 9dba8da03d ([BugFix] fix pk concurrent apply issue (#51225)) virtual void on_shutdown() {} +protected: void _gen_tablet_path(); TabletState _state; diff --git a/be/src/storage/tablet.h b/be/src/storage/tablet.h index 720ddd8ab4d5d..2905e29b3f8b1 100644 --- a/be/src/storage/tablet.h +++ b/be/src/storage/tablet.h @@ -331,7 +331,6 @@ class Tablet : public BaseTablet { // set true when start to drop tablet. only set in `TabletManager::drop_tablet` right now void set_is_dropping(bool is_dropping) { _is_dropping = is_dropping; } -protected: void on_shutdown() override; private: diff --git a/be/src/storage/tablet_manager.cpp b/be/src/storage/tablet_manager.cpp index 15a7ef5fce9ed..5a0aaae3591f5 100644 --- a/be/src/storage/tablet_manager.cpp +++ b/be/src/storage/tablet_manager.cpp @@ -437,6 +437,16 @@ Status TabletManager::drop_tablet(TTabletId tablet_id, TabletDropFlag flag) { _add_shutdown_tablet_unlocked(tablet_id, std::move(drop_info)); } else { DCHECK_EQ(kKeepMetaAndFiles, flag); + { + // If the tablet is the primary key table, there might still be ongoing apply tasks. + // We should stop the background apply tasks before deleting it; otherwise, if a new tablet is created, + // there could be a scenario where apply tasks are executed simultaneously. + // e.g. + // 1. drop and clone a new tablet with the same tablet_id. + // 2. compact rocksdb meta and reload tablet again. + std::unique_lock l(dropped_tablet->get_header_lock()); + dropped_tablet->on_shutdown(); + } } // erase tablet from tablet map { diff --git a/be/src/storage/tablet_updates.h b/be/src/storage/tablet_updates.h index e2f0cc7a8e3cb..3f4edfb84392e 100644 --- a/be/src/storage/tablet_updates.h +++ b/be/src/storage/tablet_updates.h @@ -494,6 +494,7 @@ class TabletUpdates { _last_compaction_time_ms = UnixMillis(); } } + bool is_apply_stop() { return _apply_stopped.load(); } bool compaction_running() { return _compaction_running; } diff --git a/be/test/storage/tablet_updates_test.cpp b/be/test/storage/tablet_updates_test.cpp index 9716c23fe092f..80453f803f4ff 100644 --- a/be/test/storage/tablet_updates_test.cpp +++ b/be/test/storage/tablet_updates_test.cpp @@ -3731,4 +3731,11 @@ TEST_F(TabletUpdatesTest, test_get_compaction_status) { test_horizontal_compaction(false, true); } +TEST_F(TabletUpdatesTest, test_drop_tablet_with_keep_meta_and_files) { + _tablet = create_tablet(rand(), rand()); + ASSERT_FALSE(_tablet->updates()->is_apply_stop()); + StorageEngine::instance()->tablet_manager()->drop_tablet(_tablet->tablet_id(), kKeepMetaAndFiles); + ASSERT_TRUE(_tablet->updates()->is_apply_stop()); +} + } // namespace starrocks From 27f502d87b3a92fde97b6ffddcf9abd5630938ac Mon Sep 17 00:00:00 2001 From: sevev Date: Tue, 24 Sep 2024 10:08:01 +0800 Subject: [PATCH 2/2] reslove conflict Signed-off-by: sevev --- be/src/storage/base_tablet.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/be/src/storage/base_tablet.h b/be/src/storage/base_tablet.h index d8321f1b9d3e8..07466a0471841 100644 --- a/be/src/storage/base_tablet.h +++ b/be/src/storage/base_tablet.h @@ -115,12 +115,6 @@ class BaseTablet : public std::enable_shared_from_this { return flag; } -<<<<<<< HEAD -protected: -======= - virtual size_t num_rows() const = 0; - ->>>>>>> 9dba8da03d ([BugFix] fix pk concurrent apply issue (#51225)) virtual void on_shutdown() {} protected: