diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index 694b7d6db84d8e..b207cc4c5ad22b 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -201,7 +201,7 @@ void Merger::vertical_split_columns(const TabletSchema& tablet_schema, << ", delete_sign_idx=" << delete_sign_idx; // for duplicate no keys if (!key_columns.empty()) { - column_groups->emplace_back(std::move(key_columns)); + column_groups->emplace_back(key_columns); } std::vector value_columns; diff --git a/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out b/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out index 3edad07916c9b5..a401117495f32d 100644 --- a/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out +++ b/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out @@ -67,3 +67,71 @@ 2 bason bason hate pear 99 3 bason bason hate pear 99 +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + +-- !sql -- +1 bason bason hate pear 99 +2 bason bason hate pear 99 +3 bason bason hate pear 99 + diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy index 013d5a558111e7..e07618be6c1370 100644 --- a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy +++ b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy @@ -112,69 +112,22 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index") } } - boolean invertedIndexCompactionEnable = false - boolean has_update_be_config = false - try { - String backend_id; - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List + def run_test = { table_name -> - for (Object ele in (List) configList) { - assert ele instanceof List - if (((List) ele)[0] == "inverted_index_compaction_enable") { - invertedIndexCompactionEnable = Boolean.parseBoolean(((List) ele)[2]) - logger.info("inverted_index_compaction_enable: ${((List) ele)[2]}") - } - if (((List) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) - logger.info("disable_auto_compaction: ${((List) ele)[2]}") - } - } - set_be_config.call("inverted_index_compaction_enable", "true") - has_update_be_config = true - // check updated config - check_config.call("inverted_index_compaction_enable", "true"); + sql """ INSERT INTO ${table_name} VALUES (1, "andy", "andy love apple", 100); """ + sql """ INSERT INTO ${table_name} VALUES (1, "bason", "bason hate pear", 99); """ + sql """ INSERT INTO ${table_name} VALUES (2, "andy", "andy love apple", 100); """ + sql """ INSERT INTO ${table_name} VALUES (2, "bason", "bason hate pear", 99); """ + sql """ INSERT INTO ${table_name} VALUES (3, "andy", "andy love apple", 100); """ + sql """ INSERT INTO ${table_name} VALUES (3, "bason", "bason hate pear", 99); """ - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(255) NULL, - `hobbies` text NULL, - `score` int(11) NULL, - index index_name (name) using inverted, - index index_hobbies (hobbies) using inverted properties("parser"="english"), - index index_score (score) using inverted - ) ENGINE=OLAP - UNIQUE KEY(`id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1", - "disable_auto_compaction" = "true", - "enable_unique_key_merge_on_write" = "true" - ); - """ - - sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """ - sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """ - sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """ - sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """ - - qt_sql """ select * from ${tableName} order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """ //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ + def tablets = sql_return_maparray """ show tablets from ${table_name}; """ def dedup_tablets = deduplicate_tablets(tablets) // In the p0 testing environment, there are no expected operations such as scaling down BE (backend) services @@ -189,7 +142,7 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index") int rowsetCount = get_rowset_count.call(tablets); assert (rowsetCount == 7 * replicaNum) - // trigger full compactions for all tablets in ${tableName} + // trigger full compactions for all tablets in ${table_name} trigger_full_compaction_on_tablets.call(tablets) // wait for full compaction done @@ -203,23 +156,23 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index") assert (rowsetCount == 1 * replicaNum) } - qt_sql """ select * from ${tableName} order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """ // insert more data and trigger full compaction again - sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """ - sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """ - sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """ - sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """ - - qt_sql """ select * from ${tableName} order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """ + sql """ INSERT INTO ${table_name} VALUES (1, "andy", "andy love apple", 100); """ + sql """ INSERT INTO ${table_name} VALUES (1, "bason", "bason hate pear", 99); """ + sql """ INSERT INTO ${table_name} VALUES (2, "andy", "andy love apple", 100); """ + sql """ INSERT INTO ${table_name} VALUES (2, "bason", "bason hate pear", 99); """ + sql """ INSERT INTO ${table_name} VALUES (3, "andy", "andy love apple", 100); """ + sql """ INSERT INTO ${table_name} VALUES (3, "bason", "bason hate pear", 99); """ + + qt_sql """ select * from ${table_name} order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """ rowsetCount = get_rowset_count.call(tablets); if (isCloudMode) { @@ -228,7 +181,7 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index") assert (rowsetCount == 7 * replicaNum) } - // trigger full compactions for all tablets in ${tableName} + // trigger full compactions for all tablets in ${table_name} trigger_full_compaction_on_tablets.call(tablets) // wait for full compaction done @@ -242,10 +195,86 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index") assert (rowsetCount == 1 * replicaNum) } - qt_sql """ select * from ${tableName} order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """ - qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """ + qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """ + } + + boolean invertedIndexCompactionEnable = false + boolean has_update_be_config = false + try { + String backend_id; + backend_id = backendId_to_backendIP.keySet()[0] + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) + + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "inverted_index_compaction_enable") { + invertedIndexCompactionEnable = Boolean.parseBoolean(((List) ele)[2]) + logger.info("inverted_index_compaction_enable: ${((List) ele)[2]}") + } + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + logger.info("disable_auto_compaction: ${((List) ele)[2]}") + } + } + set_be_config.call("inverted_index_compaction_enable", "true") + has_update_be_config = true + // check updated config + check_config.call("inverted_index_compaction_enable", "true"); + + sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NULL, + `name` varchar(255) NULL, + `hobbies` text NULL, + `score` int(11) NULL, + index index_name (name) using inverted, + index index_hobbies (hobbies) using inverted properties("parser"="english"), + index index_score (score) using inverted + ) ENGINE=OLAP + UNIQUE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + run_test.call(tableName) + + tableName = "test_index_compaction_unique_keys_arr_cluster_key" + sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NULL, + `name` varchar(255) NULL, + `hobbies` text NULL, + `score` int(11) NULL, + index index_name (name) using inverted, + index index_hobbies (hobbies) using inverted properties("parser"="english"), + index index_score (score) using inverted + ) ENGINE=OLAP + UNIQUE KEY(`id`) + CLUSTER BY (`score`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ); + """ + run_test.call(tableName) } finally { if (has_update_be_config) {