[Improve](complex type) delete fe config disable_nested_complex_type (#…

…36255) same with #36255
apache · Jun 18, 2024 · c43a20f · c43a20f
1 parent 6a9fe27
commit c43a20f
Show file tree

Hide file tree

Showing 13 changed files with 5 additions and 92 deletions.
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2084,17 +2084,6 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true)
     public static boolean disable_datev1  = true;
 
-    /**
-     * Now we not fully support array/struct/map nesting complex type in many situation,
-     * so just disable creating nesting complex data type when create table.
-     * We can make it able after we fully support
-     */
-    @ConfField(mutable = true, masterOnly = true, description = {
-            "当前默认设置为 true，不支持建表时创建复杂类型(array/struct/map)嵌套复杂类型, 仅支持array类型自身嵌套。",
-            "Now default set to true, not support create complex type(array/struct/map) nested complex type "
-                    + "when we create table, only support array type nested array"})
-    public static boolean disable_nested_complex_type  = true;
-
     /*
      * This variable indicates the number of digits by which to increase the scale
      * of the result of division operations performed with the `/` operator. The

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -28,7 +28,6 @@
 import org.apache.doris.catalog.StructType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
 import org.apache.doris.thrift.TColumnDesc;
 import org.apache.doris.thrift.TPrimitiveType;
 
@@ -129,18 +128,10 @@ private void analyze(Type type) throws AnalysisException {
                 Type itemType = ((ArrayType) type).getItemType();
                 if (itemType instanceof ScalarType) {
                     analyzeNestedType(type, (ScalarType) itemType);
-                } else if (Config.disable_nested_complex_type && !(itemType instanceof ArrayType)) {
-                    // now we can array nesting array
-                    throw new AnalysisException("Unsupported data type: ARRAY<" + itemType.toSql() + ">");
                 }
             }
             if (type.isMapType()) {
                 MapType mt = (MapType) type;
-                if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
-                        || !(mt.getValueType() instanceof ScalarType))) {
-                    throw new AnalysisException("Unsupported data type: MAP<" + mt.getKeyType().toSql() + ","
-                        + mt.getValueType().toSql() + ">");
-                }
                 if (mt.getKeyType() instanceof ScalarType) {
                     analyzeNestedType(type, (ScalarType) mt.getKeyType());
                 }
@@ -159,8 +150,6 @@ private void analyze(Type type) throws AnalysisException {
                             throw new AnalysisException("Duplicate field name "
                                     + field.getName() + " in struct " + type.toSql());
                         }
-                    } else if (Config.disable_nested_complex_type) {
-                        throw new AnalysisException("Unsupported field type: " + fieldType.toSql() + " for STRUCT");
                     }
                 }
             }

diff --git a/...ion-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out b/...ion-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
@@ -2,9 +2,6 @@
 -- !sql --
 0
 
--- !sql --
-0
-
 -- !sql --
 2	{1:[0.35815922932906263, 0.0011899152357573994, 0.28749219850167373, 0.93512930168283781, 0.1552584991620739, 0.73308976093672584, 0.52815960653805338, 0.92936404769642733, 0.89084215295591418, 0.21986459138832559], 2:[0.25076205844319044, 0.54003619330849928, 0.70661164863002113, 0.99472899095144263, 0.078314941019622886, 0.44206845606243961, 0.30857433265707379, 0.048661247184825784, 0.76954870938240083, 0.27253204080482074], 3:[0.687619207224633, 0.81614795840545351, 0.88663481647216158, 0.38735602510379774, 0.80082741637394084, 0.39139397466930215, 0.81747099649629329, 0.17689152877700398, 0.293284479800614, 0.4831799656491037], 4:[0.91830437289915945, 0.073632718338386915, 0.77375510933714242, 0.16816732397102252, 0.43277238052192457, 0.44208980569126877, 0.79186698174595249, 0.16564847296516494, 0.0094630056071454138, 0.35501997374061678], 5:[0.46246293630195257, 0.068748494114144409, 0.99666952760980676, 0.19382320901198535, 0.13487672467906897, 0.33765716511811372, 0.94048529885220122, 0.49731102964787222, 0.32349099284636484, 0.48597339317230759], 6:[0.085073215003979574, 0.28401725721857973, 0.20014761214390719, 0.20524657901039456, 0.11392467553698971, 0.24438834044276414, 0.49910871763701081, 0.33495152703896491, 0.46452858927426788, 0.390595446136895], 7:[0.20961408068486054, 0.67796351365684493, 0.088745389947127551, 0.12660368488966578, 0.505554324463536, 0.27743440100778682, 0.13385620029678247, 0.59106155590166665, 0.10571357063523412, 0.042619186963761324], 8:[0.68279814520834636, 0.94813249012442957, 0.94054706738722227, 0.47041093695670555, 0.58451395465060019, 0.16449033849974781, 0.14272344659751923, 0.66091032176672571, 0.4394562483212725, 0.16905962729691759]}	{"G72Qcx-UBjfOr1-Pis-7YGo":{"UsPMIs-ipxhEnU-1EG-RJpe":0.22741640776012562, "gcn7fm-ILbMhQ6-fcz-TndT":0.36370276228098763, "o8dgBn-1bM26Wz-8SJ-xW6u":0.55671646501523719, "wav6ZA-780SwLJ-Vj3-KCv5":0.8665187582647581}}
 

diff --git a/...-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy b/...-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
@@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
 suite("two_level_nestedtypes_with_s3data") {
     sql """set enable_nereids_planner=false"""
     sql """ set enable_fallback_to_original_planner=true;"""
-    sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
     // this test case aim to test one-level nested type with s3 data
 
 

diff --git a/regression-test/suites/datatype_p0/nested_types/load.groovy b/regression-test/suites/datatype_p0/nested_types/load.groovy
@@ -17,7 +17,7 @@
 
 suite("load") {
     // ddl begin
-    sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
+
     sql """set enable_nereids_planner=false"""
     def dataFile = """test_scalar_types_100.csv"""
 

diff --git a/...t/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy b/...t/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
@@ -24,7 +24,8 @@ suite("test_nested_types_insert_into_with_literal", "p0") {
     sql 'use regression_test_datatype_p0_nested_types'
     // old planner does not support cast empty
     sql 'set enable_nereids_planner=false'
-    sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
+//    sql 'set enable_nereids_planner=true'
+//    sql 'set enable_fallback_to_original_planner=false'
 
     def table_names = [
             "two_level_array_array_a",

diff --git a/...n-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy b/...n-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
@@ -24,7 +24,6 @@ suite("test_nested_types_insert_into_with_s3", "p0") {
     sql 'use regression_test_datatype_p0_nested_types'
     sql 'set enable_nereids_planner=false'
     sql 'set max_allowed_packet=4194304'
-    sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
 
     String ak = getS3AK()
     String sk = getS3SK()

diff --git a/...est/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy b/...est/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
@@ -25,7 +25,6 @@ suite("test_nestedtypes_csv_insert_into_with_s3", "p0") {
     sql 'set enable_nereids_planner=false'
     sql 'set max_allowed_packet=4194304'
     sql 'set topn_opt_limit_threshold=10000'
-    sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
 
     String ak = getS3AK()
     String sk = getS3SK()

diff --git a/...st/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy b/...st/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
@@ -25,7 +25,6 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") {
     sql 'set enable_nereids_planner=false'
     sql 'set max_allowed_packet=4194304'
     sql 'set topn_opt_limit_threshold=10000'
-    sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
 
     String ak = getS3AK()
     String sk = getS3SK()

diff --git a/...est/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy b/...est/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
@@ -43,7 +43,7 @@ suite("test_local_tvf_with_complex_type_insertinto_doris","external,hive,tvf,ext
         }
     }
 
-    qt_sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
+
 
     // create doris table
     sql """ DROP TABLE IF EXISTS ${table_name} """

diff --git a/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy b/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
@@ -17,7 +17,6 @@
 
 suite("test_load_with_map_nested_array", "p0") {
     def tableName = "test_load_with_map_nested_array"
-    sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
     sql """ DROP TABLE IF EXISTS ${tableName} """
 
     sql """

diff --git a/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy b/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
@@ -20,7 +20,7 @@ suite("map_agg_nested_insert_doris", "p0") {
     def tb_doris = "test_map_agg_nested_insert_target"
     sql "DROP TABLE IF EXISTS `${tb_base}`;"
     sql "DROP TABLE IF EXISTS `${tb_doris}`;"
-    sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
+
 
     sql """
         CREATE TABLE `${tb_base}` (

diff --git a/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy b/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
@@ -137,64 +137,6 @@ suite("test_nested_complex_switch", "query") {
         sql "DROP TABLE IF EXISTS ${testTable_m}"
         sql "DROP TABLE IF EXISTS ${testTable_a}"
         sql "DROP TABLE IF EXISTS ${testTable_s}"
-        sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'true')"
-
-        // map
-        test {
-            sql sql_m_s
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,STRUCT<f1:TINYINT>>"
-        }
-
-        test {
-            sql sql_m_a
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<ARRAY<INT>,TEXT>"
-        }
-
-        test {
-            sql sql_m_m
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,MAP<TEXT,INT>>"
-        }
-
-        // array
-        test {
-            sql sql_a_s
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<STRUCT<f1:TINYINT>>"
-        }
-
-
-        test {
-            sql sql_a_m
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<MAP<TEXT,INT>>"
-        }
-
-        // struct
-        test {
-            sql sql_s_s
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: STRUCT<f11:BOOLEAN> for STRUCT"
-        }
-
-        test {
-            sql sql_s_a
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: ARRAY<INT> for STRUCT"
-        }
-
-        test {
-            sql sql_s_m
-            exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: MAP<BOOLEAN,TINYINT> for STRUCT"
-        }
-
-    } finally {
-        try_sql("DROP TABLE IF EXISTS ${testTable_m}")
-        try_sql("DROP TABLE IF EXISTS ${testTable_a}")
-        try_sql("DROP TABLE IF EXISTS ${testTable_s}")
-    }
-
-    try {
-        sql "DROP TABLE IF EXISTS ${testTable_m}"
-        sql "DROP TABLE IF EXISTS ${testTable_a}"
-        sql "DROP TABLE IF EXISTS ${testTable_s}"
-        sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"
-
 
         // map
         test {
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,9 +2,6 @@ @@
     -- !sql --
-    -- !sql --
     -- !sql --
 	{1:[0.35815922932906263, 0.0011899152357573994, 0.28749219850167373, 0.93512930168283781, 0.1552584991620739, 0.73308976093672584, 0.52815960653805338, 0.92936404769642733, 0.89084215295591418, 0.21986459138832559], 2:[0.25076205844319044, 0.54003619330849928, 0.70661164863002113, 0.99472899095144263, 0.078314941019622886, 0.44206845606243961, 0.30857433265707379, 0.048661247184825784, 0.76954870938240083, 0.27253204080482074], 3:[0.687619207224633, 0.81614795840545351, 0.88663481647216158, 0.38735602510379774, 0.80082741637394084, 0.39139397466930215, 0.81747099649629329, 0.17689152877700398, 0.293284479800614, 0.4831799656491037], 4:[0.91830437289915945, 0.073632718338386915, 0.77375510933714242, 0.16816732397102252, 0.43277238052192457, 0.44208980569126877, 0.79186698174595249, 0.16564847296516494, 0.0094630056071454138, 0.35501997374061678], 5:[0.46246293630195257, 0.068748494114144409, 0.99666952760980676, 0.19382320901198535, 0.13487672467906897, 0.33765716511811372, 0.94048529885220122, 0.49731102964787222, 0.32349099284636484, 0.48597339317230759], 6:[0.085073215003979574, 0.28401725721857973, 0.20014761214390719, 0.20524657901039456, 0.11392467553698971, 0.24438834044276414, 0.49910871763701081, 0.33495152703896491, 0.46452858927426788, 0.390595446136895], 7:[0.20961408068486054, 0.67796351365684493, 0.088745389947127551, 0.12660368488966578, 0.505554324463536, 0.27743440100778682, 0.13385620029678247, 0.59106155590166665, 0.10571357063523412, 0.042619186963761324], 8:[0.68279814520834636, 0.94813249012442957, 0.94054706738722227, 0.47041093695670555, 0.58451395465060019, 0.16449033849974781, 0.14272344659751923, 0.66091032176672571, 0.4394562483212725, 0.16905962729691759]}	{"G72Qcx-UBjfOr1-Pis-7YGo":{"UsPMIs-ipxhEnU-1EG-RJpe":0.22741640776012562, "gcn7fm-ILbMhQ6-fcz-TndT":0.36370276228098763, "o8dgBn-1bM26Wz-8SJ-xW6u":0.55671646501523719, "wav6ZA-780SwLJ-Vj3-KCv5":0.8665187582647581}}
@@ Expand Down @@