From 032c6a2051e6f7f936830bab35c373b35d5e76d7 Mon Sep 17 00:00:00 2001 From: wyb Date: Sat, 20 Jul 2024 18:25:07 +0800 Subject: [PATCH 1/3] [UT] Fix lake compaction UT Signed-off-by: wyb --- be/test/storage/lake/compaction_task_test.cpp | 102 +++++++++++++----- be/test/storage/lake/compaction_test_utils.h | 4 +- 2 files changed, 78 insertions(+), 28 deletions(-) diff --git a/be/test/storage/lake/compaction_task_test.cpp b/be/test/storage/lake/compaction_task_test.cpp index 7f26660372862..fe5a36b7a9494 100644 --- a/be/test/storage/lake/compaction_task_test.cpp +++ b/be/test/storage/lake/compaction_task_test.cpp @@ -52,6 +52,15 @@ class LakeCompactionTest : public TestBase, testing::WithParamInterface(task.get()) != nullptr); } } + +protected: + void SetUp() override { + config::enable_size_tiered_compaction_strategy = GetParam().enable_size_tiered_compaction_strategy; + config::min_cumulative_compaction_num_singleton_deltas = 1; + clear_and_init_test_dir(); + } + + void TearDown() override { remove_test_dir_ignore_error(); } }; class LakeDuplicateKeyCompactionTest : public LakeCompactionTest { @@ -67,13 +76,10 @@ class LakeDuplicateKeyCompactionTest : public LakeCompactionTest { constexpr static const int kChunkSize = 12; void SetUp() override { - config::enable_size_tiered_compaction_strategy = false; - clear_and_init_test_dir(); + LakeCompactionTest::SetUp(); CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata)); } - void TearDown() override { remove_test_dir_ignore_error(); } - Chunk generate_data(int64_t chunk_size) { std::vector v0(chunk_size); std::vector v1(chunk_size); @@ -160,16 +166,28 @@ TEST_P(LakeDuplicateKeyCompactionTest, test1) { ASSERT_EQ(kChunkSize * 3, read(version)); ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version)); - ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + if (GetParam().enable_size_tiered_compaction_strategy) { + ASSERT_EQ(0, new_tablet_metadata->cumulative_point()); + } else { + ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + } ASSERT_EQ(1, new_tablet_metadata->rowsets_size()); } INSTANTIATE_TEST_SUITE_P(LakeDuplicateKeyCompactionTest, LakeDuplicateKeyCompactionTest, - ::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5}, - CompactionParam{VERTICAL_COMPACTION, 1}), + ::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = false}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = false}), to_string_param_name); -TEST_F(LakeDuplicateKeyCompactionTest, test_empty_tablet) { +TEST_P(LakeDuplicateKeyCompactionTest, test_empty_tablet) { auto version = 1; ASSERT_EQ(0, read(version)); @@ -197,12 +215,10 @@ class LakeDuplicateKeyOverlapSegmentsCompactionTest : public LakeCompactionTest constexpr static const int kChunkSize = 12; void SetUp() override { - clear_and_init_test_dir(); + LakeCompactionTest::SetUp(); CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata)); } - void TearDown() override { remove_test_dir_ignore_error(); } - Chunk generate_data(int64_t chunk_size) { std::vector v0(chunk_size); std::vector v1(chunk_size); @@ -289,7 +305,7 @@ TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) { check_task(task); auto st = task->execute(CompactionTask::kCancelledFn); EXPECT_EQ(0, task_context->progress.value()); - EXPECT_TRUE(st.is_cancelled()) << st; + EXPECT_TRUE(st.is_aborted()) << st; } // Completed compaction task without error { @@ -306,7 +322,11 @@ TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) { // check metadata ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version)); - ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + if (GetParam().enable_size_tiered_compaction_strategy) { + ASSERT_EQ(0, new_tablet_metadata->cumulative_point()); + } else { + ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + } ASSERT_EQ(1, new_tablet_metadata->rowsets_size()); ASSERT_EQ(1, new_tablet_metadata->rowsets(0).segments_size()); @@ -330,8 +350,16 @@ TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) { } INSTANTIATE_TEST_SUITE_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, LakeDuplicateKeyOverlapSegmentsCompactionTest, - ::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5}, - CompactionParam{VERTICAL_COMPACTION, 1}), + ::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = false}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = false}), to_string_param_name); class LakeUniqueKeyCompactionTest : public LakeCompactionTest { @@ -347,12 +375,10 @@ class LakeUniqueKeyCompactionTest : public LakeCompactionTest { constexpr static const int kChunkSize = 12; void SetUp() override { - clear_and_init_test_dir(); + LakeCompactionTest::SetUp(); CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata)); } - void TearDown() override { remove_test_dir_ignore_error(); } - Chunk generate_data(int64_t chunk_size) { std::vector v0(chunk_size); std::vector v1(chunk_size); @@ -439,13 +465,25 @@ TEST_P(LakeUniqueKeyCompactionTest, test1) { ASSERT_EQ(kChunkSize, read(version)); ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version)); - ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + if (GetParam().enable_size_tiered_compaction_strategy) { + ASSERT_EQ(0, new_tablet_metadata->cumulative_point()); + } else { + ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + } ASSERT_EQ(1, new_tablet_metadata->rowsets_size()); } INSTANTIATE_TEST_SUITE_P(LakeUniqueKeyCompactionTest, LakeUniqueKeyCompactionTest, - ::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5}, - CompactionParam{VERTICAL_COMPACTION, 1}), + ::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = false}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = false}), to_string_param_name); class LakeUniqueKeyCompactionWithDeleteTest : public LakeCompactionTest { @@ -461,12 +499,10 @@ class LakeUniqueKeyCompactionWithDeleteTest : public LakeCompactionTest { constexpr static const int kChunkSize = 12; void SetUp() override { - clear_and_init_test_dir(); + LakeCompactionTest::SetUp(); CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata)); } - void TearDown() override { remove_test_dir_ignore_error(); } - Chunk generate_data(int64_t chunk_size) { std::vector v0(chunk_size); std::vector v1(chunk_size); @@ -577,13 +613,25 @@ TEST_P(LakeUniqueKeyCompactionWithDeleteTest, test_base_compaction_with_delete) ASSERT_EQ(kChunkSize - 4, read(version)); ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version)); - ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + if (GetParam().enable_size_tiered_compaction_strategy) { + ASSERT_EQ(0, new_tablet_metadata->cumulative_point()); + } else { + ASSERT_EQ(1, new_tablet_metadata->cumulative_point()); + } ASSERT_EQ(1, new_tablet_metadata->rowsets_size()); } INSTANTIATE_TEST_SUITE_P(LakeUniqueKeyCompactionWithDeleteTest, LakeUniqueKeyCompactionWithDeleteTest, - ::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5}, - CompactionParam{VERTICAL_COMPACTION, 1}), + ::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = HORIZONTAL_COMPACTION, + .enable_size_tiered_compaction_strategy = false}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = true}, + CompactionParam{.algorithm = VERTICAL_COMPACTION, + .vertical_compaction_max_columns_per_group = 1, + .enable_size_tiered_compaction_strategy = false}), to_string_param_name); } // namespace starrocks::lake diff --git a/be/test/storage/lake/compaction_test_utils.h b/be/test/storage/lake/compaction_test_utils.h index afdbb125c3b1f..38f6810408f32 100644 --- a/be/test/storage/lake/compaction_test_utils.h +++ b/be/test/storage/lake/compaction_test_utils.h @@ -26,13 +26,15 @@ struct CompactionParam { uint32_t vertical_compaction_max_columns_per_group = 5; bool enable_persistent_index = false; PersistentIndexTypePB persistent_index_type = PersistentIndexTypePB::LOCAL; + bool enable_size_tiered_compaction_strategy = true; }; static std::string to_string_param_name(const testing::TestParamInfo& info) { std::stringstream ss; ss << CompactionUtils::compaction_algorithm_to_string(info.param.algorithm) << "_" << info.param.vertical_compaction_max_columns_per_group << "_" << info.param.enable_persistent_index << "_" - << PersistentIndexTypePB_Name(info.param.persistent_index_type); + << PersistentIndexTypePB_Name(info.param.persistent_index_type) << "_" + << info.param.enable_size_tiered_compaction_strategy; return ss.str(); } From 28eff3e5b1a6956e2b4421b71fc8dc8774c44f54 Mon Sep 17 00:00:00 2001 From: wyb Date: Mon, 22 Jul 2024 10:11:01 +0800 Subject: [PATCH 2/3] Fix comment Signed-off-by: wyb --- be/test/storage/lake/compaction_task_test.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/be/test/storage/lake/compaction_task_test.cpp b/be/test/storage/lake/compaction_task_test.cpp index fe5a36b7a9494..7ed6a96e7a66b 100644 --- a/be/test/storage/lake/compaction_task_test.cpp +++ b/be/test/storage/lake/compaction_task_test.cpp @@ -56,11 +56,23 @@ class LakeCompactionTest : public TestBase, testing::WithParamInterface(kChunkSize); @@ -264,7 +275,6 @@ class LakeDuplicateKeyOverlapSegmentsCompactionTest : public LakeCompactionTest }; TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) { - config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group; // Prepare data for writing auto chunk0 = generate_data(kChunkSize); auto indexes = std::vector(kChunkSize); @@ -424,7 +434,6 @@ class LakeUniqueKeyCompactionTest : public LakeCompactionTest { }; TEST_P(LakeUniqueKeyCompactionTest, test1) { - config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group; // Prepare data for writing auto chunk0 = generate_data(kChunkSize); auto indexes = std::vector(kChunkSize); @@ -548,7 +557,6 @@ class LakeUniqueKeyCompactionWithDeleteTest : public LakeCompactionTest { }; TEST_P(LakeUniqueKeyCompactionWithDeleteTest, test_base_compaction_with_delete) { - config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group; // Prepare data for writing auto chunk0 = generate_data(kChunkSize); auto indexes = std::vector(kChunkSize); From bdeadaf7a8f2e96a5f4b4204bfb05eec7faadeba Mon Sep 17 00:00:00 2001 From: wyb Date: Mon, 22 Jul 2024 10:18:08 +0800 Subject: [PATCH 3/3] Fix format Signed-off-by: wyb --- be/test/storage/lake/compaction_task_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/test/storage/lake/compaction_task_test.cpp b/be/test/storage/lake/compaction_task_test.cpp index 7ed6a96e7a66b..60f6ae1a9c84a 100644 --- a/be/test/storage/lake/compaction_task_test.cpp +++ b/be/test/storage/lake/compaction_task_test.cpp @@ -72,7 +72,6 @@ class LakeCompactionTest : public TestBase, testing::WithParamInterface