Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Improve](complex type) delete fe config disable_nested_complex_type #36255

Merged
merged 5 commits into from
Jun 18, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions fe/fe-common/src/main/java/org/apache/doris/common/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -2084,17 +2084,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean disable_datev1 = true;

/**
* Now we not fully support array/struct/map nesting complex type in many situation,
* so just disable creating nesting complex data type when create table.
* We can make it able after we fully support
*/
@ConfField(mutable = true, masterOnly = true, description = {
"当前默认设置为 true,不支持建表时创建复杂类型(array/struct/map)嵌套复杂类型, 仅支持array类型自身嵌套。",
"Now default set to true, not support create complex type(array/struct/map) nested complex type "
+ "when we create table, only support array type nested array"})
public static boolean disable_nested_complex_type = true;

/*
* This variable indicates the number of digits by which to increase the scale
* of the result of division operations performed with the `/` operator. The
Expand Down
11 changes: 0 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.thrift.TColumnDesc;
import org.apache.doris.thrift.TPrimitiveType;

Expand Down Expand Up @@ -129,18 +128,10 @@ private void analyze(Type type) throws AnalysisException {
Type itemType = ((ArrayType) type).getItemType();
if (itemType instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) itemType);
} else if (Config.disable_nested_complex_type && !(itemType instanceof ArrayType)) {
// now we can array nesting array
throw new AnalysisException("Unsupported data type: ARRAY<" + itemType.toSql() + ">");
}
}
if (type.isMapType()) {
MapType mt = (MapType) type;
if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
|| !(mt.getValueType() instanceof ScalarType))) {
throw new AnalysisException("Unsupported data type: MAP<" + mt.getKeyType().toSql() + ","
+ mt.getValueType().toSql() + ">");
}
if (mt.getKeyType() instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) mt.getKeyType());
}
Expand All @@ -159,8 +150,6 @@ private void analyze(Type type) throws AnalysisException {
throw new AnalysisException("Duplicate field name "
+ field.getName() + " in struct " + type.toSql());
}
} else if (Config.disable_nested_complex_type) {
throw new AnalysisException("Unsupported field type: " + fieldType.toSql() + " for STRUCT");
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@
-- !sql --
0

-- !sql --
0

-- !sql --
2 {1:[0.35815922932906263, 0.0011899152357573994, 0.28749219850167373, 0.93512930168283781, 0.1552584991620739, 0.73308976093672584, 0.52815960653805338, 0.92936404769642733, 0.89084215295591418, 0.21986459138832559], 2:[0.25076205844319044, 0.54003619330849928, 0.70661164863002113, 0.99472899095144263, 0.078314941019622886, 0.44206845606243961, 0.30857433265707379, 0.048661247184825784, 0.76954870938240083, 0.27253204080482074], 3:[0.687619207224633, 0.81614795840545351, 0.88663481647216158, 0.38735602510379774, 0.80082741637394084, 0.39139397466930215, 0.81747099649629329, 0.17689152877700398, 0.293284479800614, 0.4831799656491037], 4:[0.91830437289915945, 0.073632718338386915, 0.77375510933714242, 0.16816732397102252, 0.43277238052192457, 0.44208980569126877, 0.79186698174595249, 0.16564847296516494, 0.0094630056071454138, 0.35501997374061678], 5:[0.46246293630195257, 0.068748494114144409, 0.99666952760980676, 0.19382320901198535, 0.13487672467906897, 0.33765716511811372, 0.94048529885220122, 0.49731102964787222, 0.32349099284636484, 0.48597339317230759], 6:[0.085073215003979574, 0.28401725721857973, 0.20014761214390719, 0.20524657901039456, 0.11392467553698971, 0.24438834044276414, 0.49910871763701081, 0.33495152703896491, 0.46452858927426788, 0.390595446136895], 7:[0.20961408068486054, 0.67796351365684493, 0.088745389947127551, 0.12660368488966578, 0.505554324463536, 0.27743440100778682, 0.13385620029678247, 0.59106155590166665, 0.10571357063523412, 0.042619186963761324], 8:[0.68279814520834636, 0.94813249012442957, 0.94054706738722227, 0.47041093695670555, 0.58451395465060019, 0.16449033849974781, 0.14272344659751923, 0.66091032176672571, 0.4394562483212725, 0.16905962729691759]} {"G72Qcx-UBjfOr1-Pis-7YGo":{"UsPMIs-ipxhEnU-1EG-RJpe":0.22741640776012562, "gcn7fm-ILbMhQ6-fcz-TndT":0.36370276228098763, "o8dgBn-1bM26Wz-8SJ-xW6u":0.55671646501523719, "wav6ZA-780SwLJ-Vj3-KCv5":0.8665187582647581}}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
suite("two_level_nestedtypes_with_s3data") {
sql """set enable_nereids_planner=false"""
sql """ set enable_fallback_to_original_planner=true;"""
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
// this test case aim to test one-level nested type with s3 data


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

suite("load") {
// ddl begin
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""

sql """set enable_nereids_planner=false"""
def dataFile = """test_scalar_types_100.csv"""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_nested_types_insert_into_with_literal", "p0") {
sql 'use regression_test_datatype_p0_nested_types'
// old planner does not support cast empty
sql 'set enable_nereids_planner=false'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
// sql 'set enable_nereids_planner=false'
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'

def table_names = [
"two_level_array_array_a",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ suite("test_nested_types_insert_into_with_s3", "p0") {
sql 'use regression_test_datatype_p0_nested_types'
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""

String ak = getS3AK()
String sk = getS3SK()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ suite("test_nestedtypes_csv_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""

String ak = getS3AK()
String sk = getS3SK()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""

String ak = getS3AK()
String sk = getS3SK()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ suite("test_local_tvf_with_complex_type_insertinto_doris","external,hive,tvf,ext
}
}

qt_sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""


// create doris table
sql """ DROP TABLE IF EXISTS ${table_name} """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

suite("test_load_with_map_nested_array", "p0") {
def tableName = "test_load_with_map_nested_array"
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
sql """ DROP TABLE IF EXISTS ${tableName} """

sql """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ suite("map_agg_nested_insert_doris", "p0") {
def tb_doris = "test_map_agg_nested_insert_target"
sql "DROP TABLE IF EXISTS `${tb_base}`;"
sql "DROP TABLE IF EXISTS `${tb_doris}`;"
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """


sql """
CREATE TABLE `${tb_base}` (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,64 +137,6 @@ suite("test_nested_complex_switch", "query") {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'true')"

// map
test {
sql sql_m_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,STRUCT<f1:TINYINT>>"
}

test {
sql sql_m_a
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<ARRAY<INT>,TEXT>"
}

test {
sql sql_m_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,MAP<TEXT,INT>>"
}

// array
test {
sql sql_a_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<STRUCT<f1:TINYINT>>"
}


test {
sql sql_a_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<MAP<TEXT,INT>>"
}

// struct
test {
sql sql_s_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: STRUCT<f11:BOOLEAN> for STRUCT"
}

test {
sql sql_s_a
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: ARRAY<INT> for STRUCT"
}

test {
sql sql_s_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: MAP<BOOLEAN,TINYINT> for STRUCT"
}

} finally {
try_sql("DROP TABLE IF EXISTS ${testTable_m}")
try_sql("DROP TABLE IF EXISTS ${testTable_a}")
try_sql("DROP TABLE IF EXISTS ${testTable_s}")
}

try {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"


// map
test {
Expand Down
Loading