[improve](schema-change) support nested type with varchar type to support length growing

fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java

@@ -511,6 +511,41 @@ public boolean isDecimalV3OrContainsDecimalV3() {
         return false;
     }
 
+    // This method defines the char type or complex type nested char type
+    // to support the schema-change behavior of length growth.
+    public boolean isSupportSchemaChangeForCharType(Type other) {
+        if ((this.getPrimitiveType() == PrimitiveType.VARCHAR && other.getPrimitiveType() == PrimitiveType.VARCHAR) || (
+                this.getPrimitiveType() == PrimitiveType.CHAR && other.getPrimitiveType() == PrimitiveType.VARCHAR) || (
+                this.getPrimitiveType() == PrimitiveType.CHAR && other.getPrimitiveType() == PrimitiveType.CHAR)) {
+            if (this.getLength() > other.getLength()) {
+                return false;
+            } else {
+                return true;
+            }
+        }
+        if (this.isStructType() && other.isStructType()) {
+            StructType thisStructType = (StructType) this;
+            StructType otherStructType = (StructType) other;
+            if (thisStructType.getFields().size() != otherStructType.getFields().size()) {
+                return false;
+            }
+            for (int i = 0; i < thisStructType.getFields().size(); i++) {
+                if (!thisStructType.getFields().get(i).getType().isSupportSchemaChangeForCharType(
+                        otherStructType.getFields().get(i).getType())) {
+                    return false;
+                }
+            }
+            return true;
+        } else if (this.isArrayType() && other.isArrayType()) {
+            return ((ArrayType) this).getItemType().isSupportSchemaChangeForCharType(((ArrayType) other).getItemType());
+        } else if (this.isMapType() && other.isMapType()) {
+            return ((MapType) this).getKeyType().isSupportSchemaChangeForCharType(((MapType) other).getKeyType())
+                    &&
+                    ((MapType) this).getValueType().isSupportSchemaChangeForCharType(((MapType) other).getValueType());
+        }
+        return false;
+    }
+
     public boolean isDecimalV3() {
         return isScalarType(PrimitiveType.DECIMAL32) || isScalarType(PrimitiveType.DECIMAL64)
                 || isScalarType(PrimitiveType.DECIMAL128) || isScalarType(PrimitiveType.DECIMAL256);

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java

@@ -627,8 +627,9 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol
                 if (!col.equals(modColumn)) {
                     typeChanged = true;
                     // TODO:the case where columnPos is not empty has not been considered
-                    if (columnPos == null && col.getDataType() == PrimitiveType.VARCHAR
-                            && modColumn.getDataType() == PrimitiveType.VARCHAR) {
+                    if (columnPos == null && ((col.getDataType() == PrimitiveType.VARCHAR
+                            && modColumn.getDataType() == PrimitiveType.VARCHAR)
+                            || col.getDataType().isComplexType())) {
                         col.checkSchemaChangeAllowed(modColumn);
                         lightSchemaChange = olapTable.getEnableLightSchemaChange();
                     }

fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java

@@ -844,11 +844,6 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException {
             throw new DdlException("Dest column name is empty");
         }
 
-        // now nested type can only support change order
-        if (type.isComplexType() && !type.equals(other.type)) {
-            throw new DdlException("Can not change " + type + " to " + other);
-        }
-
         if (!ColumnType.isSchemaChangeAllowed(type, other.type)) {
             throw new DdlException("Can not change " + getDataType() + " to " + other.getDataType());
         }
@@ -885,12 +880,10 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException {
             }
         }
 
-        if ((getDataType() == PrimitiveType.VARCHAR && other.getDataType() == PrimitiveType.VARCHAR) || (
-                getDataType() == PrimitiveType.CHAR && other.getDataType() == PrimitiveType.VARCHAR) || (
-                getDataType() == PrimitiveType.CHAR && other.getDataType() == PrimitiveType.CHAR)) {
-            if (getStrLen() > other.getStrLen()) {
-                throw new DdlException("Cannot shorten string length");
-            }
+        // Nested types only support changing the order and increasing the length of the nested char type
+        // Char-type only support length growing
+        if (!type.isSupportSchemaChangeForCharType(other.type)) {
+            throw new DdlException("Cannot shorten string length for type " + type.toSql() + " to " + other.toSql());
         }
 
         // now we support convert decimal to varchar type

regression-test/data/schema_change_p0/test_varchar_sc_in_complex.out

@@ -0,0 +1,31 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sc_origin --
+0	["2025-01-01"]	{"amory":"better"}	{"col":"commiter"}
+1	["2025-01-02"]	{"doris":"better"}	{"col":"amory"}
+
+-- !sc_after --
+0	["2025-01-01"]	{"amory":"better"}	{"col":"commiter"}
+1	["2025-01-02"]	{"doris":"better"}	{"col":"amory"}
+2	["2025-01-03-22-33"]	{"doris":"better"}	{"col":"amory"}
+
+-- !sc_origin --
+0	["2025-01-01"]	{"amory":"better"}	{"col":"commiter"}
+1	["2025-01-02"]	{"doris":"better"}	{"col":"amory"}
+2	["2025-01-03-22-33"]	{"doris":"better"}	{"col":"amory"}
+
+-- !sc_after --
+0	["2025-01-01"]	{"amory":"better"}	{"col":"commiter"}
+1	["2025-01-02"]	{"doris":"better"}	{"col":"amory"}
+2	["2025-01-03-22-33"]	{"doris":"better"}	{"col":"amory"}
+
+-- !sc_origin --
+0	["2025-01-01"]	{"amory":"better"}	{"col":"commiter"}
+1	["2025-01-02"]	{"doris":"better"}	{"col":"amory"}
+2	["2025-01-03-22-33"]	{"doris":"better"}	{"col":"amory"}
+
+-- !sc_after --
+0	["2025-01-01"]	{"amory":"better"}	{"col":"commiter"}
+1	["2025-01-02"]	{"doris":"better"}	{"col":"amory"}
+2	["2025-01-03-22-33"]	{"doris":"better"}	{"col":"amory"}
+4	["2025-01-03-22-33"]	{"doris":"better"}	{"col":"amoryIsBetter"}
+

regression-test/suites/schema_change_p0/test_varchar_sc_in_complex.groovy

@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+suite ("test_varchar_sc_in_complex") {
+    def getJobState = { tableName ->
+         def jobStateResult = sql """  SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+         return jobStateResult[0][9]
+    }
+
+    def tableName = "test_varchar_sc_in_complex"
+
+    try {
+
+        String backend_id;
+        def backendId_to_backendIP = [:]
+        def backendId_to_backendHttpPort = [:]
+        getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+                CREATE TABLE IF NOT EXISTS ${tableName} (
+                    `c0` LARGEINT NOT NULL,
+                    `c_a` ARRAY<VARCHAR(10)>,
+                    `c_m` MAP<VARCHAR(10),VARCHAR(10)>,
+                    `c_s` STRUCT<col:VARCHAR(10)>
+                ) DISTRIBUTED BY HASH(c0) BUCKETS 1
+                PROPERTIES ( "replication_num" = "1", "light_schema_change" = "true" )
+            """
+
+        sql """ insert into ${tableName} values
+                (0,['2025-01-01'], {'amory':'better'}, named_struct('col','commiter'));
+            """
+        sql """ insert into ${tableName} values
+                (1,['2025-01-02'], {'doris':'better'}, named_struct('col','amory'));
+            """
+        // this can be insert but with cut off the left string to 10
+        test {
+            sql """ insert into ${tableName} values
+                (11, ['2025-01-03-22-33'], {'doris111111111':'better2222222222'}, named_struct('col','amoryIsBetter'));
+            """
+            exception "Insert has filtered data in strict mode"
+        }
+
+        test {
+            sql """ alter table ${tableName} modify column c_a array<varchar(3)>
+                """
+            exception "Cannot shorten string length"
+        }
+
+        test {
+            sql """ alter table ${tableName} modify column c_m map<varchar(3),varchar(3)>
+                """
+            exception "Cannot shorten string length"
+        }
+
+        test {
+            sql """ alter table ${tableName} modify column c_s struct<col:varchar(3)>
+                """
+            exception "Cannot shorten string length"
+        }
+
+
+        sql """ alter table ${tableName} modify column c_a array<varchar(20)> """
+        int max_try_secs = 300
+        Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> {
+            String result = getJobState(tableName)
+            if (result == "FINISHED") {
+                return true;
+            }
+            return false;
+        });
+
+        String[][] res = sql """ desc ${tableName} """
+        logger.info(res[1][1])
+        // array varchar(10)
+        assertEquals(res[1][1].toLowerCase(),"array<varchar(20)>")
+
+        qt_sc_origin " select * from ${tableName} order by c0; "
+
+        // then insert some data to modified array with varchar(20)
+        sql """ insert into ${tableName} values
+                (2,['2025-01-03-22-33'], {'doris':'better'}, named_struct('col','amory'));
+            """
+        qt_sc_after " select * from ${tableName} order by c0; "
+
+        // map varchar(10)
+        sql """ alter table ${tableName} modify column c_m map<varchar(20),varchar(20)> """
+        Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> {
+            String result = getJobState(tableName)
+            if (result == "FINISHED") {
+                return true;
+            }
+            return false;
+        });
+
+        res = sql """ desc ${tableName} """
+        logger.info(res[2][1])
+        assertEquals(res[2][1].toLowerCase(),"map<varchar(20),varchar(20)>")
+
+        // insert some data to modified map with varchar(20)
+        qt_sc_origin " select * from ${tableName} order by c0; "
+        sql """ insert into ts values
+                (3,['2025-01-03-22-33'], {'doris111111111':'better2222222222'}, named_struct('col','amory'));
+            """
+        qt_sc_after " select * from ${tableName} order by c0; "
+
+        // struct varchar(10)
+        sql """ alter table ${tableName} modify column c_s struct<col:varchar(20)> """
+        Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> {
+            String result = getJobState(tableName)
+            if (result == "FINISHED") {
+                return true;
+            }
+            return false;
+        });
+
+        res = sql """ desc ${tableName} """
+        logger.info(res[3][1])
+        assertEquals(res[3][1].toLowerCase(),"struct<col:varchar(20)>")
+
+        // insert some data to modified struct with varchar(20)
+        qt_sc_origin " select * from ${tableName} order by c0; "
+        sql """ insert into ${tableName} values
+                (4,['2025-01-03-22-33'], {'doris':'better'}, named_struct('col','amoryIsBetter'));
+            """
+        qt_sc_after " select * from ${tableName} order by c0; "
+    } finally {
+         try_sql("DROP TABLE IF EXISTS ${tableName}")
+    }
+
+}