Skip to content

Commit

Permalink
[UT] Fix lake compaction UT (#48648)
Browse files Browse the repository at this point in the history 
Signed-off-by: wyb <[email protected]>
(cherry picked from commit 507625f)

# Conflicts:
#	be/test/storage/lake/compaction_test_utils.h
  • Loading branch information
@wyb @mergify
wyb authored and mergify[bot] committed Jul 22, 2024
1 parent cc775d7 commit 03e8ce4
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 31 deletions.
117 changes: 86 additions & 31 deletions be/test/storage/lake/compaction_task_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,26 @@ class LakeCompactionTest : public TestBase, testing::WithParamInterface<Compacti
ASSERT_TRUE(dynamic_cast<VerticalCompactionTask*>(task.get()) != nullptr);
}
}

protected:
void SetUp() override {
config::enable_size_tiered_compaction_strategy = GetParam().enable_size_tiered_compaction_strategy;
config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group;
config::min_cumulative_compaction_num_singleton_deltas = 1;
clear_and_init_test_dir();
}

void TearDown() override {
remove_test_dir_ignore_error();
config::enable_size_tiered_compaction_strategy = _enable_size_tiered_compaction_strategy;
config::vertical_compaction_max_columns_per_group = _vertical_compaction_max_columns_per_group;
config::min_cumulative_compaction_num_singleton_deltas = _min_cumulative_compaction_num_singleton_deltas;
}

private:
bool _enable_size_tiered_compaction_strategy = config::enable_size_tiered_compaction_strategy;
int64_t _vertical_compaction_max_columns_per_group = config::vertical_compaction_max_columns_per_group;
int64_t _min_cumulative_compaction_num_singleton_deltas = config::min_cumulative_compaction_num_singleton_deltas;
};

class LakeDuplicateKeyCompactionTest : public LakeCompactionTest {
Expand All @@ -68,13 +88,10 @@ class LakeDuplicateKeyCompactionTest : public LakeCompactionTest {
constexpr static const int kChunkSize = 12;

void SetUp() override {
config::enable_size_tiered_compaction_strategy = false;
clear_and_init_test_dir();
LakeCompactionTest::SetUp();
CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata));
}

void TearDown() override { remove_test_dir_ignore_error(); }

Chunk generate_data(int64_t chunk_size) {
std::vector<int> v0(chunk_size);
std::vector<int> v1(chunk_size);
Expand Down Expand Up @@ -120,7 +137,6 @@ class LakeDuplicateKeyCompactionTest : public LakeCompactionTest {
};

TEST_P(LakeDuplicateKeyCompactionTest, test1) {
config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group;
// Prepare data for writing
auto chunk0 = generate_data(kChunkSize);
auto indexes = std::vector<uint32_t>(kChunkSize);
Expand Down Expand Up @@ -161,16 +177,28 @@ TEST_P(LakeDuplicateKeyCompactionTest, test1) {
ASSERT_EQ(kChunkSize * 3, read(version));

ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version));
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
if (GetParam().enable_size_tiered_compaction_strategy) {
ASSERT_EQ(0, new_tablet_metadata->cumulative_point());
} else {
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
}
ASSERT_EQ(1, new_tablet_metadata->rowsets_size());
}

INSTANTIATE_TEST_SUITE_P(LakeDuplicateKeyCompactionTest, LakeDuplicateKeyCompactionTest,
::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5},
CompactionParam{VERTICAL_COMPACTION, 1}),
::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = false},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = false}),
to_string_param_name);

TEST_F(LakeDuplicateKeyCompactionTest, test_empty_tablet) {
TEST_P(LakeDuplicateKeyCompactionTest, test_empty_tablet) {
auto version = 1;
ASSERT_EQ(0, read(version));

Expand Down Expand Up @@ -198,12 +226,10 @@ class LakeDuplicateKeyOverlapSegmentsCompactionTest : public LakeCompactionTest
constexpr static const int kChunkSize = 12;

void SetUp() override {
clear_and_init_test_dir();
LakeCompactionTest::SetUp();
CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata));
}

void TearDown() override { remove_test_dir_ignore_error(); }

Chunk generate_data(int64_t chunk_size) {
std::vector<int> v0(chunk_size);
std::vector<int> v1(chunk_size);
Expand Down Expand Up @@ -249,7 +275,6 @@ class LakeDuplicateKeyOverlapSegmentsCompactionTest : public LakeCompactionTest
};

TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) {
config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group;
// Prepare data for writing
auto chunk0 = generate_data(kChunkSize);
auto indexes = std::vector<uint32_t>(kChunkSize);
Expand Down Expand Up @@ -290,7 +315,7 @@ TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) {
check_task(task);
auto st = task->execute(CompactionTask::kCancelledFn);
EXPECT_EQ(0, task_context->progress.value());
EXPECT_TRUE(st.is_cancelled()) << st;
EXPECT_TRUE(st.is_aborted()) << st;
}
// Completed compaction task without error
{
Expand All @@ -307,7 +332,11 @@ TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) {

// check metadata
ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version));
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
if (GetParam().enable_size_tiered_compaction_strategy) {
ASSERT_EQ(0, new_tablet_metadata->cumulative_point());
} else {
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
}
ASSERT_EQ(1, new_tablet_metadata->rowsets_size());
ASSERT_EQ(1, new_tablet_metadata->rowsets(0).segments_size());

Expand All @@ -331,8 +360,16 @@ TEST_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, test) {
}

INSTANTIATE_TEST_SUITE_P(LakeDuplicateKeyOverlapSegmentsCompactionTest, LakeDuplicateKeyOverlapSegmentsCompactionTest,
::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5},
CompactionParam{VERTICAL_COMPACTION, 1}),
::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = false},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = false}),
to_string_param_name);

class LakeUniqueKeyCompactionTest : public LakeCompactionTest {
Expand All @@ -348,12 +385,10 @@ class LakeUniqueKeyCompactionTest : public LakeCompactionTest {
constexpr static const int kChunkSize = 12;

void SetUp() override {
clear_and_init_test_dir();
LakeCompactionTest::SetUp();
CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata));
}

void TearDown() override { remove_test_dir_ignore_error(); }

Chunk generate_data(int64_t chunk_size) {
std::vector<int> v0(chunk_size);
std::vector<int> v1(chunk_size);
Expand Down Expand Up @@ -399,7 +434,6 @@ class LakeUniqueKeyCompactionTest : public LakeCompactionTest {
};

TEST_P(LakeUniqueKeyCompactionTest, test1) {
config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group;
// Prepare data for writing
auto chunk0 = generate_data(kChunkSize);
auto indexes = std::vector<uint32_t>(kChunkSize);
Expand Down Expand Up @@ -440,13 +474,25 @@ TEST_P(LakeUniqueKeyCompactionTest, test1) {
ASSERT_EQ(kChunkSize, read(version));

ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version));
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
if (GetParam().enable_size_tiered_compaction_strategy) {
ASSERT_EQ(0, new_tablet_metadata->cumulative_point());
} else {
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
}
ASSERT_EQ(1, new_tablet_metadata->rowsets_size());
}

INSTANTIATE_TEST_SUITE_P(LakeUniqueKeyCompactionTest, LakeUniqueKeyCompactionTest,
::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5},
CompactionParam{VERTICAL_COMPACTION, 1}),
::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = false},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = false}),
to_string_param_name);

class LakeUniqueKeyCompactionWithDeleteTest : public LakeCompactionTest {
Expand All @@ -462,12 +508,10 @@ class LakeUniqueKeyCompactionWithDeleteTest : public LakeCompactionTest {
constexpr static const int kChunkSize = 12;

void SetUp() override {
clear_and_init_test_dir();
LakeCompactionTest::SetUp();
CHECK_OK(_tablet_mgr->put_tablet_metadata(*_tablet_metadata));
}

void TearDown() override { remove_test_dir_ignore_error(); }

Chunk generate_data(int64_t chunk_size) {
std::vector<int> v0(chunk_size);
std::vector<int> v1(chunk_size);
Expand Down Expand Up @@ -513,7 +557,6 @@ class LakeUniqueKeyCompactionWithDeleteTest : public LakeCompactionTest {
};

TEST_P(LakeUniqueKeyCompactionWithDeleteTest, test_base_compaction_with_delete) {
config::vertical_compaction_max_columns_per_group = GetParam().vertical_compaction_max_columns_per_group;
// Prepare data for writing
auto chunk0 = generate_data(kChunkSize);
auto indexes = std::vector<uint32_t>(kChunkSize);
Expand Down Expand Up @@ -578,13 +621,25 @@ TEST_P(LakeUniqueKeyCompactionWithDeleteTest, test_base_compaction_with_delete)
ASSERT_EQ(kChunkSize - 4, read(version));

ASSIGN_OR_ABORT(auto new_tablet_metadata, _tablet_mgr->get_tablet_metadata(tablet_id, version));
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
if (GetParam().enable_size_tiered_compaction_strategy) {
ASSERT_EQ(0, new_tablet_metadata->cumulative_point());
} else {
ASSERT_EQ(1, new_tablet_metadata->cumulative_point());
}
ASSERT_EQ(1, new_tablet_metadata->rowsets_size());
}

INSTANTIATE_TEST_SUITE_P(LakeUniqueKeyCompactionWithDeleteTest, LakeUniqueKeyCompactionWithDeleteTest,
::testing::Values(CompactionParam{HORIZONTAL_COMPACTION, 5},
CompactionParam{VERTICAL_COMPACTION, 1}),
::testing::Values(CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = HORIZONTAL_COMPACTION,
.enable_size_tiered_compaction_strategy = false},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = true},
CompactionParam{.algorithm = VERTICAL_COMPACTION,
.vertical_compaction_max_columns_per_group = 1,
.enable_size_tiered_compaction_strategy = false}),
to_string_param_name);

} // namespace starrocks::lake
Expand Down
11 changes: 11 additions & 0 deletions be/test/storage/lake/compaction_test_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,23 @@ struct CompactionParam {
CompactionAlgorithm algorithm = HORIZONTAL_COMPACTION;
uint32_t vertical_compaction_max_columns_per_group = 5;
bool enable_persistent_index = false;
<<<<<<< HEAD
=======
PersistentIndexTypePB persistent_index_type = PersistentIndexTypePB::LOCAL;
bool enable_size_tiered_compaction_strategy = true;
>>>>>>> 507625f004 ([UT] Fix lake compaction UT (#48648))
};

static std::string to_string_param_name(const testing::TestParamInfo<CompactionParam>& info) {
std::stringstream ss;
ss << CompactionUtils::compaction_algorithm_to_string(info.param.algorithm) << "_"
<<<<<<< HEAD
<< info.param.vertical_compaction_max_columns_per_group << "_" << info.param.enable_persistent_index;
=======
<< info.param.vertical_compaction_max_columns_per_group << "_" << info.param.enable_persistent_index << "_"
<< PersistentIndexTypePB_Name(info.param.persistent_index_type) << "_"
<< info.param.enable_size_tiered_compaction_strategy;
>>>>>>> 507625f004 ([UT] Fix lake compaction UT (#48648))
return ss.str();
}

Expand Down

0 comments on commit 03e8ce4

Please sign in to comment.