From 1b27acd333e5f9e22f6f9ea619dfc7c6d56902a5 Mon Sep 17 00:00:00 2001 From: Sun Chenyang Date: Tue, 12 Nov 2024 21:00:39 +0800 Subject: [PATCH 1/4] [fix] (inverted index) Disallow variant columns from using inverted index format v1 (#43599) Problem Summary: 1. When the inverted index of a variant column uses storage format v1, schema changes can cause some segments to lack corresponding index files. 2. By using storage format v2 for inverted indexes, all indexes correspond to a single file, and the corresponding files will always exist regardless of whether the variant includes subcolumn indexes. When creating an inverted index for a variant column, file format v1 is not supported --- .../doris/alter/SchemaChangeHandler.java | 7 +- .../doris/analysis/CreateTableStmt.java | 8 +- .../org/apache/doris/analysis/IndexDef.java | 12 +- .../plans/commands/info/CreateTableInfo.java | 12 +- .../plans/commands/info/IndexDefinition.java | 10 +- .../org/apache/doris/qe/SessionVariable.java | 13 ++ .../apache/doris/analysis/IndexDefTest.java | 13 ++ .../plans/commands/IndexDefinitionTest.java | 46 +++++++ ...paction_with_variant_inverted_index.groovy | 2 + .../inverted_index_p0/load/test_insert.groovy | 85 +++++++++++++ .../load/test_stream_load.groovy | 118 ++++++++++++++++++ ...index_file_http_action_with_variant.groovy | 2 + .../test_variant_index_format_v1.groovy | 3 +- .../load_p2/test_single_replica_load.groovy | 2 + regression-test/suites/variant_p0/load.groovy | 2 +- .../variant_p0/with_index/var_index.groovy | 60 +++++++++ 16 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java create mode 100644 regression-test/suites/inverted_index_p0/load/test_insert.groovy create mode 100644 regression-test/suites/inverted_index_p0/load/test_stream_load.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 72148a5c2824b7..2017e1c2cd60d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2586,12 +2586,15 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab + " ) already exist."); } } - + boolean disableInvertedIndexV1ForVariant = olapTable.getInvertedIndexFileStorageFormat() + == TInvertedIndexFileStorageFormat.V1 && ConnectContext.get().getSessionVariable() + .getDisableInvertedIndexV1ForVaraint(); for (String col : indexDef.getColumns()) { Column column = olapTable.getColumn(col); if (column != null) { indexDef.checkColumn(column, olapTable.getKeysType(), - olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite()); + olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), + disableInvertedIndexV1ForVariant); indexDef.getColumnUniqueIds().add(column.getUniqueId()); } else { throw new DdlException("index column does not exist in table. invalid column: " + col); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index fc36416b299656..bf7581a348edf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -42,6 +42,7 @@ import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -561,7 +562,9 @@ public void analyze(Analyzer analyzer) throws UserException { if (CollectionUtils.isNotEmpty(indexDefs)) { Set distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); Set>> distinctCol = new HashSet<>(); - + boolean disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( + new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1 + && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint(); for (IndexDef indexDef : indexDefs) { indexDef.analyze(); if (!engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) { @@ -571,7 +574,8 @@ public void analyze(Analyzer analyzer) throws UserException { boolean found = false; for (Column column : columns) { if (column.getName().equalsIgnoreCase(indexColName)) { - indexDef.checkColumn(column, getKeysDesc().getKeysType(), enableUniqueKeyMergeOnWrite); + indexDef.checkColumn(column, getKeysDesc().getKeysType(), + enableUniqueKeyMergeOnWrite, disableInvertedIndexV1ForVariant); found = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index d19fdb379197e8..4e4002fe51afe7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -213,8 +213,8 @@ public boolean isInvertedIndex() { return (this.indexType == IndexType.INVERTED); } - public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite) - throws AnalysisException { + public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite, + boolean disableInvertedIndexV1ForVariant) throws AnalysisException { if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) { String indexColName = column.getName(); @@ -229,6 +229,14 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + indexName); } + + // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS. + // when the subcolumn type changes, it may result in missing files, causing link file failure. + if (colType.isVariantType() && disableInvertedIndexV1ForVariant) { + throw new AnalysisException(colType + " is not supported in inverted index format V1," + + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + + "or upgrade to a newer version"); + } if (!column.isKey()) { if (keysType == KeysType.AGG_KEYS) { throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 28437c49973c0c..58cbada4846757 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -51,6 +51,7 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -534,6 +535,14 @@ public void validate(ConnectContext ctx) { if (!indexes.isEmpty()) { Set distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); Set>> distinctCol = new HashSet<>(); + boolean disableInvertedIndexV1ForVariant = false; + try { + disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( + new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1 + && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint(); + } catch (Exception e) { + throw new AnalysisException(e.getMessage(), e.getCause()); + } for (IndexDefinition indexDef : indexes) { indexDef.validate(); @@ -545,7 +554,8 @@ public void validate(ConnectContext ctx) { boolean found = false; for (ColumnDefinition column : columns) { if (column.getName().equalsIgnoreCase(indexColName)) { - indexDef.checkColumn(column, keysType, isEnableMergeOnWrite); + indexDef.checkColumn(column, keysType, isEnableMergeOnWrite, + disableInvertedIndexV1ForVariant); found = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index cd9c19c25d4bd7..9a893d94f6da0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -94,7 +94,7 @@ public IndexDefinition(String name, List cols, String indexTypeName, * checkColumn */ public void checkColumn(ColumnDefinition column, KeysType keysType, - boolean enableUniqueKeyMergeOnWrite) throws AnalysisException { + boolean enableUniqueKeyMergeOnWrite, boolean disableInvertedIndexV1ForVariant) throws AnalysisException { if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) { String indexColName = column.getName(); @@ -110,6 +110,14 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + name); } + + // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS. + // when the subcolumn type changes, it may result in missing files, causing link file failure. + if (colType.isVariantType() && disableInvertedIndexV1ForVariant) { + throw new AnalysisException(colType + " is not supported in inverted index format V1," + + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + + "or upgrade to a newer version"); + } if (!column.isKey()) { if (keysType == KeysType.AGG_KEYS) { throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index dabb8817b231b7..e08d8d779fa51a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -634,6 +634,8 @@ public class SessionVariable implements Serializable, Writable { public static final String MAX_FETCH_REMOTE_TABLET_COUNT = "max_fetch_remote_schema_tablet_count"; + public static final String DISABLE_INVERTED_INDEX_V1_FOR_VARIANT = "disable_inverted_index_v1_for_variant"; + // CLOUD_VARIABLES_BEGIN public static final String CLOUD_CLUSTER = "cloud_cluster"; public static final String DISABLE_EMPTY_PARTITION_PRUNE = "disable_empty_partition_prune"; @@ -1166,6 +1168,9 @@ public enum IgnoreSplitType { @VariableMgr.VarAttr(name = WAIT_FULL_BLOCK_SCHEDULE_TIMES) public int waitFullBlockScheduleTimes = 2; + @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT) + private boolean disableInvertedIndexV1ForVaraint = true; + public int getBeNumberForTest() { return beNumberForTest; } @@ -4415,4 +4420,12 @@ public TSerdeDialect getSerdeDialect() { public boolean isEnableCooldownReplicaAffinity() { return enableCooldownReplicaAffinity; } + + public void setDisableInvertedIndexV1ForVaraint(boolean disableInvertedIndexV1ForVaraint) { + this.disableInvertedIndexV1ForVaraint = disableInvertedIndexV1ForVaraint; + } + + public boolean getDisableInvertedIndexV1ForVaraint() { + return disableInvertedIndexV1ForVaraint; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java index deccb435b71de5..56c78e2430a5ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java @@ -17,6 +17,9 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Lists; @@ -60,6 +63,16 @@ public void testAnalyzeExpection() throws AnalysisException { } catch (AnalysisException e) { Assert.assertTrue(e instanceof AnalysisException); } + try { + def = new IndexDef("variant_index", false, Lists.newArrayList("col1"), + IndexDef.IndexType.INVERTED, null, "comment"); + boolean isIndexFormatV1 = true; + def.checkColumn(new Column("col1", PrimitiveType.VARIANT), KeysType.UNIQUE_KEYS, true, isIndexFormatV1); + Assert.fail("No exception throws."); + } catch (AnalysisException e) { + Assert.assertTrue(e instanceof AnalysisException); + Assert.assertTrue(e.getMessage().contains("not supported in inverted index format V1")); + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java new file mode 100644 index 00000000000000..0632945ee0416e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; +import org.apache.doris.nereids.types.VariantType; + +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class IndexDefinitionTest { + @Test + void testVariantIndexFormatV1() throws AnalysisException { + IndexDefinition def = new IndexDefinition("variant_index", Lists.newArrayList("col1"), "INVERTED", + null, "comment"); + try { + boolean isIndexFormatV1 = true; + def.checkColumn(new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, + null, "comment"), KeysType.UNIQUE_KEYS, true, isIndexFormatV1); + Assertions.fail("No exception throws."); + } catch (AnalysisException e) { + Assertions.assertTrue(e instanceof AnalysisException); + Assertions.assertTrue(e.getMessage().contains("not supported in inverted index format V1")); + } + } +} diff --git a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy index 788c3ad46083fd..fce01e48dddf3e 100644 --- a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy +++ b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy @@ -129,6 +129,7 @@ suite("test_single_compaction_with_variant_inverted", "p2") { sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE ${tableName} ( `id` int(11) NULL, @@ -147,6 +148,7 @@ suite("test_single_compaction_with_variant_inverted", "p2") { "compaction_policy" = "time_series" ); """ + sql """ set disable_inverted_index_v1_for_variant = true """ def tablets = sql_return_maparray """ show tablets from ${tableName}; """ diff --git a/regression-test/suites/inverted_index_p0/load/test_insert.groovy b/regression-test/suites/inverted_index_p0/load/test_insert.groovy new file mode 100644 index 00000000000000..03e7a3ccb26404 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/load/test_insert.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_insert_with_index", "p0") { + + def set_be_config = { key, value -> + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + for (String backend_id: backendId_to_backendIP.keySet()) { + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + def test = { format -> + def srcName = "src_table" + def dstName = "dst_table" + sql """ DROP TABLE IF EXISTS ${srcName}; """ + sql """ + CREATE TABLE ${srcName} ( + k bigint, + v variant, + INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`k`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k`) BUCKETS 2 + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = ${format}); + """ + + sql """insert into ${srcName} values(1, '{"a" : 123, "b" : "xxxyyy", "c" : 111999111}')""" + sql """insert into ${srcName} values(2, '{"a" : 18811, "b" : "hello world", "c" : 1181111}')""" + sql """insert into ${srcName} values(3, '{"a" : 18811, "b" : "hello wworld", "c" : 11111}')""" + sql """insert into ${srcName} values(4, '{"a" : 1234, "b" : "hello xxx world", "c" : 8181111}')""" + qt_sql_2 """select * from ${srcName} where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" + sql """insert into ${srcName} values(5, '{"a" : 123456789, "b" : 123456, "c" : 8181111}')""" + qt_sql_3 """select * from ${srcName} where cast(v["a"] as int) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 11111 order by k""" + + sql """ DROP TABLE IF EXISTS ${dstName}; """ + sql """ + CREATE TABLE ${dstName} ( + k bigint, + v variant, + INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`k`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k`) BUCKETS 2 + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = ${format}); + """ + sql """ insert into ${dstName} select * from ${srcName}""" + qt_sql_2 """select * from ${srcName} where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" + qt_sql_3 """select * from ${srcName} where cast(v["a"] as int) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 11111 order by k""" + sql """ DROP TABLE IF EXISTS ${dstName}; """ + sql """ DROP TABLE IF EXISTS ${srcName}; """ + } + + set_be_config("inverted_index_ram_dir_enable", "true") + sql """ set disable_inverted_index_v1_for_variant = false """ + test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ + test.call("V2") + set_be_config("inverted_index_ram_dir_enable", "false") + sql """ set disable_inverted_index_v1_for_variant = false """ + test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ + test.call("V2") + set_be_config("inverted_index_ram_dir_enable", "true") +} \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy new file mode 100644 index 00000000000000..1912fdcfc50c4c --- /dev/null +++ b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_stream_load_with_inverted_index_p0", "p0") { + + def set_be_config = { key, value -> + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + for (String backend_id: backendId_to_backendIP.keySet()) { + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + + def tableName = "test_stream_load_with_inverted_index" + def calc_file_crc_on_tablet = { ip, port, tablet -> + return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) + } + + def load_json_data = {table_name, file_name -> + // load the json data + streamLoad { + table "${table_name}" + + // set http request header params + set 'read_json_by_line', 'true' + set 'format', 'json' + set 'max_filter_ratio', '0.1' + set 'memtable_on_sink_node', 'true' + file file_name // import json file + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + logger.info("Stream load ${file_name} result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + def test = { format -> + sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ + CREATE TABLE ${tableName} ( + k bigint, + v variant, + INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`k`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k`) BUCKETS 10 + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "${format}", "disable_auto_compaction" = "true"); + """ + + load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") + + def tablets = sql_return_maparray """ show tablets from ${tableName}; """ + + for (def tablet in tablets) { + String tablet_id = tablet.TabletId + backend_id = tablet.BackendId + (code, out, err) = calc_file_crc_on_tablet(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run calc file: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def resultJson = parseJson(out.trim()) + assertEquals(resultJson.start_version, "0") + assertEquals(resultJson.end_version, "2") + assertEquals(resultJson.rowset_count, "2") + } + qt_sql_1 """ + select cast(v["repo"]["name"] as string) from ${tableName} where cast(v["repo"]["name"] as string) match_phrase_prefix "davesbingrewardsbot"; + """ + + sql """ DROP TABLE IF EXISTS ${tableName}; """ + } + + set_be_config("inverted_index_ram_dir_enable", "true") + sql """ set disable_inverted_index_v1_for_variant = false """ + test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ + test.call("V2") + set_be_config("inverted_index_ram_dir_enable", "false") + sql """ set disable_inverted_index_v1_for_variant = false """ + test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ + test.call("V2") + set_be_config("inverted_index_ram_dir_enable", "true") +} \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy index 71831140a996a7..a7718e3927a4a3 100644 --- a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy +++ b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy @@ -64,6 +64,7 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0" def tableName = "test_show_nested_index_file_http_action_with_variant_" + format sql "DROP TABLE IF EXISTS ${tableName}" + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( k bigint, @@ -74,6 +75,7 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0" DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "${format}"); """ + sql """ set disable_inverted_index_v1_for_variant = true """ load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") diff --git a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy index cc44ac2bb38234..5f262c8bf07775 100644 --- a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy +++ b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy @@ -60,6 +60,7 @@ suite("test_variant_index_format_v1", "p2") { def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, @@ -70,7 +71,7 @@ suite("test_variant_index_format_v1", "p2") { DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); """ - + sql """ set disable_inverted_index_v1_for_variant = true """ set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") diff --git a/regression-test/suites/load_p2/test_single_replica_load.groovy b/regression-test/suites/load_p2/test_single_replica_load.groovy index c8ea2c578eb8e4..cb4f970ac66d1f 100644 --- a/regression-test/suites/load_p2/test_single_replica_load.groovy +++ b/regression-test/suites/load_p2/test_single_replica_load.groovy @@ -51,6 +51,7 @@ suite("test_single_replica_load", "p2") { def tableName = "test_single_replica_load" sql "DROP TABLE IF EXISTS ${tableName}" + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( k bigint, @@ -61,6 +62,7 @@ suite("test_single_replica_load", "p2") { DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "2", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); """ + sql """ set disable_inverted_index_v1_for_variant = true """ load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index 955fcb40ac2652..d737a1e1c85d8f 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -371,7 +371,7 @@ suite("regression_test_variant", "p0"){ "is_being_synced" = "false", "storage_medium" = "hdd", "storage_format" = "V2", - "inverted_index_storage_format" = "V1", + "inverted_index_storage_format" = "V2", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "store_row_column" = "true", diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index ab7b41eb83ba59..f52d0e28759dd2 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -45,4 +45,64 @@ suite("regression_test_variant_var_index", "p0"){ sql """insert into var_index values(8, '{"timestamp": [123]}')""" sql """insert into var_index values(9, '{"timestamp": 17.0}'),(10, '{"timestamp": "17.0"}')""" qt_sql "select * from var_index order by k limit 10" + + sql """ set disable_inverted_index_v1_for_variant = true """ + sql "DROP TABLE IF EXISTS var_index" + try { + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant, + INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("not supported in inverted index format V1")) + } + + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + + try { + sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED""" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("not supported in inverted index format V1")) + } + + sql """ set disable_inverted_index_v1_for_variant = false """ + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant, + INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + + sql "DROP TABLE IF EXISTS var_index" + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED""" + sql """ set disable_inverted_index_v1_for_variant = true """ } \ No newline at end of file From 6d44e68d4d79194e2232ff9a920d3684fe55df12 Mon Sep 17 00:00:00 2001 From: csun5285 Date: Mon, 9 Dec 2024 12:39:52 +0800 Subject: [PATCH 2/4] fix --- .../java/org/apache/doris/alter/SchemaChangeHandler.java | 4 ++-- .../java/org/apache/doris/analysis/CreateTableStmt.java | 4 ++-- .../nereids/trees/plans/commands/info/CreateTableInfo.java | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 2017e1c2cd60d8..ebad5f24f4f922 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2586,8 +2586,8 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab + " ) already exist."); } } - boolean disableInvertedIndexV1ForVariant = olapTable.getInvertedIndexFileStorageFormat() - == TInvertedIndexFileStorageFormat.V1 && ConnectContext.get().getSessionVariable() + boolean disableInvertedIndexV1ForVariant = olapTable.getInvertedIndexStorageFormat() + == TInvertedIndexStorageFormat.V1 && ConnectContext.get().getSessionVariable() .getDisableInvertedIndexV1ForVaraint(); for (String col : indexDef.getColumns()) { Column column = olapTable.getColumn(col); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index bf7581a348edf0..333f4eabc6b8e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -562,8 +562,8 @@ public void analyze(Analyzer analyzer) throws UserException { if (CollectionUtils.isNotEmpty(indexDefs)) { Set distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); Set>> distinctCol = new HashSet<>(); - boolean disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( - new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1 + boolean disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexStorageFormat( + new HashMap<>(properties)) == TInvertedIndexStorageFormat.V1 && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint(); for (IndexDef indexDef : indexDefs) { indexDef.analyze(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 58cbada4846757..638b87c07e0776 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -51,7 +51,7 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -537,8 +537,8 @@ public void validate(ConnectContext ctx) { Set>> distinctCol = new HashSet<>(); boolean disableInvertedIndexV1ForVariant = false; try { - disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( - new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1 + disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexStorageFormat( + new HashMap<>(properties)) == TInvertedIndexStorageFormat.V1 && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint(); } catch (Exception e) { throw new AnalysisException(e.getMessage(), e.getCause()); From dffc6288604124096b4b057e90cb38539bf8928e Mon Sep 17 00:00:00 2001 From: csun5285 Date: Mon, 9 Dec 2024 12:41:57 +0800 Subject: [PATCH 3/4] remove unused pick --- .../inverted_index_p0/load/test_insert.groovy | 85 ------------- .../load/test_stream_load.groovy | 118 ------------------ 2 files changed, 203 deletions(-) delete mode 100644 regression-test/suites/inverted_index_p0/load/test_insert.groovy delete mode 100644 regression-test/suites/inverted_index_p0/load/test_stream_load.groovy diff --git a/regression-test/suites/inverted_index_p0/load/test_insert.groovy b/regression-test/suites/inverted_index_p0/load/test_insert.groovy deleted file mode 100644 index 03e7a3ccb26404..00000000000000 --- a/regression-test/suites/inverted_index_p0/load/test_insert.groovy +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_insert_with_index", "p0") { - - def set_be_config = { key, value -> - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } - def test = { format -> - def srcName = "src_table" - def dstName = "dst_table" - sql """ DROP TABLE IF EXISTS ${srcName}; """ - sql """ - CREATE TABLE ${srcName} ( - k bigint, - v variant, - INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' - ) ENGINE=OLAP - DUPLICATE KEY(`k`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k`) BUCKETS 2 - PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = ${format}); - """ - - sql """insert into ${srcName} values(1, '{"a" : 123, "b" : "xxxyyy", "c" : 111999111}')""" - sql """insert into ${srcName} values(2, '{"a" : 18811, "b" : "hello world", "c" : 1181111}')""" - sql """insert into ${srcName} values(3, '{"a" : 18811, "b" : "hello wworld", "c" : 11111}')""" - sql """insert into ${srcName} values(4, '{"a" : 1234, "b" : "hello xxx world", "c" : 8181111}')""" - qt_sql_2 """select * from ${srcName} where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" - sql """insert into ${srcName} values(5, '{"a" : 123456789, "b" : 123456, "c" : 8181111}')""" - qt_sql_3 """select * from ${srcName} where cast(v["a"] as int) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 11111 order by k""" - - sql """ DROP TABLE IF EXISTS ${dstName}; """ - sql """ - CREATE TABLE ${dstName} ( - k bigint, - v variant, - INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' - ) ENGINE=OLAP - DUPLICATE KEY(`k`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k`) BUCKETS 2 - PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = ${format}); - """ - sql """ insert into ${dstName} select * from ${srcName}""" - qt_sql_2 """select * from ${srcName} where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" - qt_sql_3 """select * from ${srcName} where cast(v["a"] as int) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 11111 order by k""" - sql """ DROP TABLE IF EXISTS ${dstName}; """ - sql """ DROP TABLE IF EXISTS ${srcName}; """ - } - - set_be_config("inverted_index_ram_dir_enable", "true") - sql """ set disable_inverted_index_v1_for_variant = false """ - test.call("V1") - sql """ set disable_inverted_index_v1_for_variant = true """ - test.call("V2") - set_be_config("inverted_index_ram_dir_enable", "false") - sql """ set disable_inverted_index_v1_for_variant = false """ - test.call("V1") - sql """ set disable_inverted_index_v1_for_variant = true """ - test.call("V2") - set_be_config("inverted_index_ram_dir_enable", "true") -} \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy deleted file mode 100644 index 1912fdcfc50c4c..00000000000000 --- a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_stream_load_with_inverted_index_p0", "p0") { - - def set_be_config = { key, value -> - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } - - def tableName = "test_stream_load_with_inverted_index" - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } - - def load_json_data = {table_name, file_name -> - // load the json data - streamLoad { - table "${table_name}" - - // set http request header params - set 'read_json_by_line', 'true' - set 'format', 'json' - set 'max_filter_ratio', '0.1' - set 'memtable_on_sink_node', 'true' - file file_name // import json file - time 10000 // limit inflight 10s - - // if declared a check callback, the default check condition will ignore. - // So you must check all condition - - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception - } - logger.info("Stream load ${file_name} result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) - assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) - } - } - } - - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - def test = { format -> - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ - CREATE TABLE ${tableName} ( - k bigint, - v variant, - INDEX idx_v (`v`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' - ) ENGINE=OLAP - DUPLICATE KEY(`k`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k`) BUCKETS 10 - PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "${format}", "disable_auto_compaction" = "true"); - """ - - load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") - - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = calc_file_crc_on_tablet(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run calc file: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def resultJson = parseJson(out.trim()) - assertEquals(resultJson.start_version, "0") - assertEquals(resultJson.end_version, "2") - assertEquals(resultJson.rowset_count, "2") - } - qt_sql_1 """ - select cast(v["repo"]["name"] as string) from ${tableName} where cast(v["repo"]["name"] as string) match_phrase_prefix "davesbingrewardsbot"; - """ - - sql """ DROP TABLE IF EXISTS ${tableName}; """ - } - - set_be_config("inverted_index_ram_dir_enable", "true") - sql """ set disable_inverted_index_v1_for_variant = false """ - test.call("V1") - sql """ set disable_inverted_index_v1_for_variant = true """ - test.call("V2") - set_be_config("inverted_index_ram_dir_enable", "false") - sql """ set disable_inverted_index_v1_for_variant = false """ - test.call("V1") - sql """ set disable_inverted_index_v1_for_variant = true """ - test.call("V2") - set_be_config("inverted_index_ram_dir_enable", "true") -} \ No newline at end of file From c36940caf035c242fb1baa1beaaa9883353d0bb6 Mon Sep 17 00:00:00 2001 From: csun5285 Date: Mon, 9 Dec 2024 14:27:14 +0800 Subject: [PATCH 4/4] fix --- .../suites/nereids_rules_p0/mv/variant/variant_mv.groovy | 6 ++---- regression-test/suites/variant_p0/mtmv.groovy | 3 +-- .../suites/variant_p0/variant_with_rowstore.groovy | 3 ++- .../suites/variant_p0/with_index/var_index.groovy | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy index 9dddde1098f684..f545276c4bc110 100644 --- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy @@ -35,8 +35,7 @@ suite("variant_mv") { repo VARIANT NULL, payload VARIANT NULL, public BOOLEAN NULL, - created_at DATETIME NULL, - INDEX idx_payload (`payload`) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'inverted index for payload' + created_at DATETIME NULL ) DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(id) BUCKETS 10 @@ -55,8 +54,7 @@ suite("variant_mv") { repo VARIANT NULL, payload VARIANT NULL, public BOOLEAN NULL, - created_at DATETIME NULL, - INDEX idx_payload (`payload`) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'inverted index for payload' + created_at DATETIME NULL ) DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(id) BUCKETS 10 diff --git a/regression-test/suites/variant_p0/mtmv.groovy b/regression-test/suites/variant_p0/mtmv.groovy index 5d26c8fcf5647a..37f97788ea9ce2 100644 --- a/regression-test/suites/variant_p0/mtmv.groovy +++ b/regression-test/suites/variant_p0/mtmv.groovy @@ -61,8 +61,7 @@ suite("regression_test_variant_mtmv"){ `repo` VARIANT NULL, `payload` VARIANT NULL, `public` BOOLEAN NULL, - `created_at` DATETIME NULL, - INDEX idx_payload (`payload`) USING INVERTED PROPERTIES("parser" = "english", "lower_case" = "true") COMMENT 'inverted index for payload' + `created_at` DATETIME NULL ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 diff --git a/regression-test/suites/variant_p0/variant_with_rowstore.groovy b/regression-test/suites/variant_p0/variant_with_rowstore.groovy index d1946b8123c04c..87724f2c9e8434 100644 --- a/regression-test/suites/variant_p0/variant_with_rowstore.groovy +++ b/regression-test/suites/variant_p0/variant_with_rowstore.groovy @@ -121,7 +121,8 @@ suite("regression_test_variant_rowstore", "variant_type"){ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "store_row_column"="true", - "replication_num" = "1" + "replication_num" = "1", + "inverted_index_storage_format"= "v2" ); """ sql """insert into table_rs_invalid_json values (1, '1|[""]')""" diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index f52d0e28759dd2..96cab9a60a10d1 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -26,7 +26,7 @@ suite("regression_test_variant_var_index", "p0"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V2"); """ sql """insert into var_index values(1, '{"a" : 123, "b" : "xxxyyy", "c" : 111999111}')"""