From b9384379de2953447b4b1bb2560ff539796f8a23 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 29 Nov 2024 10:33:54 +0800 Subject: [PATCH 1/8] [enhance](nerieds) data_format, to_monday, last_day, to_date, year implement Monotonic [enhance](nerieds) data_format, to_monday, last_day, to_date, year implement Monotonic --- .../rules/OneRangePartitionEvaluator.java | 3 + .../functions/scalar/ConvertTz.java | 6 + .../functions/scalar/DateFormat.java | 35 ++++- .../expressions/functions/scalar/LastDay.java | 18 ++- .../expressions/functions/scalar/ToDate.java | 18 ++- .../functions/scalar/ToMonday.java | 18 ++- .../expressions/functions/scalar/Year.java | 18 ++- .../partition_prune/always_mono_func.groovy | 148 ++++++++++++++++++ 8 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index eb9fd6e149160b..e605b71d22fb49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -801,6 +801,9 @@ private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult re return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func, context.rangeMap.get(func)), result.childrenResult); } + if (!func.isMonotonic()) { + return result; + } int childIndex = func.getMonotonicFunctionChildIndex(); Expression funcChild = func.child(childIndex); boolean isNullable = partitionSlotContainsNull.getOrDefault(funcChild, true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java index e0d717cd04f66a..2dbd5cbd31b8a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; @@ -87,6 +88,11 @@ public R accept(ExpressionVisitor visitor, C context) { return visitor.visitConvertTz(this, context); } + @Override + public boolean isMonotonic(Literal lower, Literal upper) { + return child(1).isConstant() && child(2).isConstant(); + } + @Override public boolean isPositive() { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java index 2cc97e7c3afecc..2f5177d23f826b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java @@ -21,7 +21,9 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; @@ -32,14 +34,16 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Set; /** * ScalarFunction 'date_format'. This class is generated by GenerateFunction. */ public class DateFormat extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, Monotonic { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) @@ -74,4 +78,33 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitDateFormat(this, context); } + + @Override + public boolean isMonotonic() { + Expression format = child(1); + if (!(format instanceof VarcharLiteral)) { + return false; + } + VarcharLiteral varcharLiteral = (VarcharLiteral) format; + String str = varcharLiteral.getValue(); + Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "%Y", "%Y-%m", + "%Y-%m-%d", "%Y-%m-%d %H", "%Y-%m-%d %H:%i", "%Y-%m-%d %H:%i:%s", "%Y-%m-%d %H:%i:%S", "%Y-%m-%d %T", + "%Y%m%d", "%Y%m"); + return monoFormat.contains(str); + } + + @Override + public boolean isPositive() { + return true; + } + + @Override + public int getMonotonicFunctionChildIndex() { + return 0; + } + + @Override + public Expression withConstantArgs(Expression literal) { + return new DateFormat(literal, child(1)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/LastDay.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/LastDay.java index bd3783abd9a963..bb45f13874778e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/LastDay.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/LastDay.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -37,7 +38,7 @@ * ScalarFunction 'last_day'. This class is generated by GenerateFunction. */ public class LastDay extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args { + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args, Monotonic { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateV2Type.INSTANCE).args(DateV2Type.INSTANCE), @@ -71,4 +72,19 @@ public R accept(ExpressionVisitor visitor, C context) { public List getSignatures() { return SIGNATURES; } + + @Override + public boolean isPositive() { + return true; + } + + @Override + public int getMonotonicFunctionChildIndex() { + return 0; + } + + @Override + public Expression withConstantArgs(Expression literal) { + return new LastDay(literal); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java index 7bda334f522710..0638ee2d33dbf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -38,7 +39,7 @@ * ScalarFunction 'to_date'. This class is generated by GenerateFunction. */ public class ToDate extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, Monotonic { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateV2Type.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT), @@ -70,4 +71,19 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitToDate(this, context); } + + @Override + public boolean isPositive() { + return true; + } + + @Override + public int getMonotonicFunctionChildIndex() { + return 0; + } + + @Override + public Expression withConstantArgs(Expression literal) { + return new ToDate(literal); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToMonday.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToMonday.java index 216b3d3ba78203..565861eacf16f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToMonday.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToMonday.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -37,7 +38,7 @@ * ScalarFunction 'to_monday'. This class is generated by GenerateFunction. */ public class ToMonday extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args { + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args, Monotonic { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateV2Type.INSTANCE).args(DateV2Type.INSTANCE), @@ -71,4 +72,19 @@ public R accept(ExpressionVisitor visitor, C context) { public List getSignatures() { return SIGNATURES; } + + @Override + public boolean isPositive() { + return true; + } + + @Override + public int getMonotonicFunctionChildIndex() { + return 0; + } + + @Override + public Expression withConstantArgs(Expression literal) { + return new ToMonday(literal); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java index 7268e1a79bd885..6f60ecd3b85dbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -37,7 +38,7 @@ * ScalarFunction 'year'. This class is generated by GenerateFunction. */ public class Year extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args { + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args, Monotonic { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(SmallIntType.INSTANCE).args(DateV2Type.INSTANCE), @@ -70,4 +71,19 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitYear(this, context); } + + @Override + public boolean isPositive() { + return true; + } + + @Override + public int getMonotonicFunctionChildIndex() { + return 0; + } + + @Override + public Expression withConstantArgs(Expression literal) { + return new Year(literal); + } } diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy new file mode 100644 index 00000000000000..722acda8abc1c7 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("always_mono_func") { + sql "drop table if exists always_mono_func" + sql """create table always_mono_func (a int, dt datetime, d date, c varchar(100)) duplicate key(a) + partition by range(dt) ( + partition p1 values less than ("2017-01-01"), + partition p2 values less than ("2018-01-01"), + partition p3 values less than ("2019-01-01"), + partition p4 values less than ("2020-01-01"), + partition p5 values less than ("2021-01-01") + ) distributed by hash(a) properties("replication_num"="1");""" + sql """INSERT INTO always_mono_func SELECT number, + date_add('2016-01-01 00:00:00', interval number month), + cast(date_add('2022-01-01 00:00:00', interval number month) as date), cast(number as varchar(65533)) FROM numbers('number'='55');""" + sql "INSERT INTO always_mono_func values(3,null,null,null);" + + // format 测试满足格式的裁剪，不满足不裁剪 + explain { + sql """select * from always_mono_func where date_format(dt, "yyyyMMdd") <"20190101" """ + contains("partitions=3/5 (p1,p2,p3)") + } + + explain { + sql """select * from always_mono_func where date_format(dt, "yyyyMMdd") <"2019-01-01" """ + contains("partitions=3/5 (p1,p2,p3)") + } + + explain { + sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd") < "2019-01-01" """ + contains("partitions=3/5 (p1,p2,p3)") + } + + explain { + sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd") > "2019-01-01" """ + contains("partitions=2/5 (p4,p5)") + } + + explain { + sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd HH:mm:ss") < "2019-01-01" """ + contains("partitions=3/5 (p1,p2,p3)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd") = "2019-01-01" """ + contains("partitions=2/5 (p3,p4)") + } + + explain { + sql """select * from always_mono_func where date_format(dt, "yyyy-MM-dd HH:mm:ss") = "2019-01-01 00:00:00" """ + contains("partitions=2/5 (p3,p4)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y") <= "2018" and date_format(dt, "%Y") > "2017" """ + contains("partitions=2/5 (p2,p3)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y") <= "2019-01-01" """ + contains("partitions=4/5 (p1,p2,p3,p4)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y-%m")>="2019-01-01" """ + contains("partitions=2/5 (p4,p5)") + } + + explain { + sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d")<>"2019-01-01" """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H")<="2019-01-01" """ + contains("partitions=3/5 (p1,p2,p3)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H:%i")> "2019-01-01" """ + contains("partitions=3/5 (p3,p4,p5)") + } + + explain { + sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H:%i:%s")> "2019-01-01 00:00:00" """ + contains("partitions=2/5 (p4,p5)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %H:%i:%S")> "2019-01-01 00:00:00" """ + contains("partitions=2/5 (p4,p5)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y-%m-%d %T") > "2019-01-01" """ + contains("partitions=3/5 (p3,p4,p5)") + } + explain { + sql """select * from always_mono_func where date_format(dt, "%Y%m") > "20190101" """ + contains("partitions=2/5 (p4,p5)") + } + + + explain { + sql """ select * from always_mono_func where last_day(dt) > "2019-01-01" """ + contains("partitions=3/5 (p3,p4,p5)") + } + + explain { + sql """ select * from always_mono_func where to_date(dt) > "2019-01-01" """ + contains("partitions=2/5 (p4,p5)") + } + + explain { + sql """ select * from always_mono_func where to_monday(dt) >= "2019-01-01" """ + contains("partitions=2/5 (p4,p5)") + } + + explain { + sql """ select * from always_mono_func where year(dt) >= 2019 """ + contains("partitions=3/5 (p3,p4,p5)") + } + + explain { + sql """select * from always_mono_func where date_format(to_monday(dt), 'yyyyMMdd') >= "20190101" """ + contains("partitions=2/5 (p4,p5)") + } + explain { + sql """select * from always_mono_func where date_format(last_day(to_monday(dt)), 'yyyyMMdd') < "20190101" """ + contains("partitions=4/5 (p1,p2,p3,p4)") + } + explain { + sql """select * from always_mono_func where date_format(last_day(to_monday(to_date(dt))), 'yyyyMMdd') <= "20190101" """ + contains("partitions=4/5 (p1,p2,p3,p4)") + } + + explain { + sql """select * from always_mono_func where date_format(date_trunc(last_day(to_monday(dt)),'month'), 'yyyyMMdd') > "20190101" """ + contains("partitions=2/5 (p4,p5)") + } +} \ No newline at end of file From 32bc61b1f4310e1af7f1d5a1f34da98da44ebf33 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 17 Dec 2024 16:29:50 +0800 Subject: [PATCH 2/8] change by comment --- .../trees/expressions/functions/scalar/DateFormat.java | 9 +++++---- .../partition_prune/always_mono_func.groovy | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java index 2f5177d23f826b..1ae130d876d965 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java @@ -53,6 +53,10 @@ public class DateFormat extends ScalarFunction FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(DateType.INSTANCE, VarcharType.SYSTEM_DEFAULT) ); + private static final Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", + "%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%d%H", "%Y-%m-%d%H:%i", "%Y-%m-%d%H:%i:%s", "%Y-%m-%d%H:%i:%S", + "%Y-%m-%d%T", "%Y%m%d", "%Y%m"); + /** * constructor with 2 arguments. */ @@ -86,10 +90,7 @@ public boolean isMonotonic() { return false; } VarcharLiteral varcharLiteral = (VarcharLiteral) format; - String str = varcharLiteral.getValue(); - Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "%Y", "%Y-%m", - "%Y-%m-%d", "%Y-%m-%d %H", "%Y-%m-%d %H:%i", "%Y-%m-%d %H:%i:%s", "%Y-%m-%d %H:%i:%S", "%Y-%m-%d %T", - "%Y%m%d", "%Y%m"); + String str = varcharLiteral.getValue().replace(" ", ""); return monoFormat.contains(str); } diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy index 722acda8abc1c7..7fc74d7eb2ac6c 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy @@ -106,7 +106,10 @@ suite("always_mono_func") { sql """select * from always_mono_func where date_format(dt, "%Y%m") > "20190101" """ contains("partitions=2/5 (p4,p5)") } - + explain { + sql """select * from always_mono_func where date_format(dt, "%Y %m") > "2019 01" """ + contains("partitions=2/5 (p4,p5)") + } explain { sql """ select * from always_mono_func where last_day(dt) > "2019-01-01" """ From d14b010233f6b584f2356d73e27c0c59435fa18b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 19 Dec 2024 15:17:59 +0800 Subject: [PATCH 3/8] add regression --- .../partition_prune/always_mono_func.groovy | 65 ++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy index 7fc74d7eb2ac6c..81b934a2c2d26c 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy @@ -125,11 +125,74 @@ suite("always_mono_func") { sql """ select * from always_mono_func where to_monday(dt) >= "2019-01-01" """ contains("partitions=2/5 (p4,p5)") } - + // year explain { sql """ select * from always_mono_func where year(dt) >= 2019 """ contains("partitions=3/5 (p3,p4,p5)") } + explain { + sql """select * from always_mono_func where year(dt) < 2019 and year(dt) > 2017""" + contains("partitions=2/5 (p2,p3)") + } + explain { + sql """select * from always_mono_func where year(dt) <2023""" + contains("partitions=5/5 (p1,p2,p3,p4,p5)") + } + // to_monday + explain { + sql """select * from always_mono_func where to_monday(dt) <'2019-01-01' """ + contains("partitions=4/5 (p1,p2,p3,p4)") + } + explain { + sql """select * from always_mono_func where to_monday(dt) ='2019-01-01' """ + contains("partitions=1/5 (p4)") + } + explain { + sql """select * from always_mono_func where to_monday(dt) >='2018-01-01' and to_monday(dt) <'2019-01-01' """ + contains("partitions=3/5 (p2,p3,p4)") + } + // to_date + explain { + sql """select * from always_mono_func where to_date(dt) <'2019-02-01' """ + contains("partitions=4/5 (p1,p2,p3,p4)") + } + explain { + sql """select * from always_mono_func where to_date(dt) <='2023-02-01' """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") + } + explain { + sql """select * from always_mono_func where to_date(dt) is null """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") + } + explain { + sql """select * from always_mono_func where to_date(dt) is not null """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") + } + explain { + sql """select * from always_mono_func where to_date(dt) >='2018-01-01' and to_date(dt) <'2019-01-01' """ + contains("partitions=2/5 (p2,p3)") + } + // last_day + explain { + sql """select * from always_mono_func where last_day(dt) <='2019-02-01' """ + contains("partitions=4/5 (p1,p2,p3,p4)") + } + explain { + sql """select * from always_mono_func where last_day(dt) <='2023-02-01' """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") + } + explain { + sql """select * from always_mono_func where last_day(dt) is null """ + contains("partitions=1/5 (p1)") + } + explain { + sql """select * from always_mono_func where last_day(dt) is not null """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") + } + explain { + sql """select * from always_mono_func where last_day(dt) >='2018-01-01' and last_day(dt) <'2019-01-01' """ + contains("partitions=2/5 (p2,p3)") + } explain { sql """select * from always_mono_func where date_format(to_monday(dt), 'yyyyMMdd') >= "20190101" """ From 9693c6fbe9396c688cc747bcf5a0505f9f2978e3 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 19 Dec 2024 18:57:11 +0800 Subject: [PATCH 4/8] add regression --- .../partition_prune/always_mono_func.groovy | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy index 81b934a2c2d26c..25e3031ff59f22 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy @@ -30,7 +30,6 @@ suite("always_mono_func") { cast(date_add('2022-01-01 00:00:00', interval number month) as date), cast(number as varchar(65533)) FROM numbers('number'='55');""" sql "INSERT INTO always_mono_func values(3,null,null,null);" - // format 测试满足格式的裁剪，不满足不裁剪 explain { sql """select * from always_mono_func where date_format(dt, "yyyyMMdd") <"20190101" """ contains("partitions=3/5 (p1,p2,p3)") @@ -112,19 +111,15 @@ suite("always_mono_func") { } explain { - sql """ select * from always_mono_func where last_day(dt) > "2019-01-01" """ - contains("partitions=3/5 (p3,p4,p5)") + sql """select * from always_mono_func where date_format(dt, "yyyyddMM") <"20190101" """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") } explain { - sql """ select * from always_mono_func where to_date(dt) > "2019-01-01" """ - contains("partitions=2/5 (p4,p5)") + sql """select * from always_mono_func where date_format(dt, "%Y-%d-%m %T") <"2019-01-01" """ + contains("partitions=5/5 (p1,p2,p3,p4,p5)") } - explain { - sql """ select * from always_mono_func where to_monday(dt) >= "2019-01-01" """ - contains("partitions=2/5 (p4,p5)") - } // year explain { sql """ select * from always_mono_func where year(dt) >= 2019 """ @@ -151,6 +146,10 @@ suite("always_mono_func") { sql """select * from always_mono_func where to_monday(dt) >='2018-01-01' and to_monday(dt) <'2019-01-01' """ contains("partitions=3/5 (p2,p3,p4)") } + explain { + sql """ select * from always_mono_func where to_monday(dt) >= "2019-01-01" """ + contains("partitions=2/5 (p4,p5)") + } // to_date explain { sql """select * from always_mono_func where to_date(dt) <'2019-02-01' """ @@ -172,6 +171,10 @@ suite("always_mono_func") { sql """select * from always_mono_func where to_date(dt) >='2018-01-01' and to_date(dt) <'2019-01-01' """ contains("partitions=2/5 (p2,p3)") } + explain { + sql """ select * from always_mono_func where to_date(dt) > "2019-01-01" """ + contains("partitions=2/5 (p4,p5)") + } // last_day explain { sql """select * from always_mono_func where last_day(dt) <='2019-02-01' """ @@ -193,6 +196,10 @@ suite("always_mono_func") { sql """select * from always_mono_func where last_day(dt) >='2018-01-01' and last_day(dt) <'2019-01-01' """ contains("partitions=2/5 (p2,p3)") } + explain { + sql """ select * from always_mono_func where last_day(dt) > "2019-01-01" """ + contains("partitions=3/5 (p3,p4,p5)") + } explain { sql """select * from always_mono_func where date_format(to_monday(dt), 'yyyyMMdd') >= "20190101" """ From 7ce376f380731867c317501a49c3a2c37fd97f74 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 19 Dec 2024 19:33:43 +0800 Subject: [PATCH 5/8] fix by comment --- .../nereids/trees/expressions/functions/scalar/DateFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java index 1ae130d876d965..4c409d431c9ccc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java @@ -53,7 +53,7 @@ public class DateFormat extends ScalarFunction FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(DateType.INSTANCE, VarcharType.SYSTEM_DEFAULT) ); - private static final Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", + private static final Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-ddHH:mm:ss", "%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%d%H", "%Y-%m-%d%H:%i", "%Y-%m-%d%H:%i:%s", "%Y-%m-%d%H:%i:%S", "%Y-%m-%d%T", "%Y%m%d", "%Y%m"); From 1f0131e1dc8ba4f557ee38d10bc5869ff70ca7b4 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 23 Dec 2024 19:53:54 +0800 Subject: [PATCH 6/8] fix complie --- .../rules/expression/rules/OneRangePartitionEvaluator.java | 3 --- .../nereids/trees/expressions/functions/scalar/DateFormat.java | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index e605b71d22fb49..eb9fd6e149160b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -801,9 +801,6 @@ private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult re return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func, context.rangeMap.get(func)), result.childrenResult); } - if (!func.isMonotonic()) { - return result; - } int childIndex = func.getMonotonicFunctionChildIndex(); Expression funcChild = func.child(childIndex); boolean isNullable = partitionSlotContainsNull.getOrDefault(funcChild, true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java index 4c409d431c9ccc..972c0c8aec709f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -84,7 +85,7 @@ public R accept(ExpressionVisitor visitor, C context) { } @Override - public boolean isMonotonic() { + public boolean isMonotonic(Literal lower, Literal upper) { Expression format = child(1); if (!(format instanceof VarcharLiteral)) { return false; From 062096d8364a2b93157b5bf1952a0979859ddb2c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 24 Dec 2024 22:26:48 +0800 Subject: [PATCH 7/8] fix regression --- .../partition_prune/always_mono_func.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy index 25e3031ff59f22..44eb0b873bbfb4 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy @@ -140,15 +140,15 @@ suite("always_mono_func") { } explain { sql """select * from always_mono_func where to_monday(dt) ='2019-01-01' """ - contains("partitions=1/5 (p4)") + contains("partitions=2/5 (p1,p4)") } explain { sql """select * from always_mono_func where to_monday(dt) >='2018-01-01' and to_monday(dt) <'2019-01-01' """ - contains("partitions=3/5 (p2,p3,p4)") + contains("partitions=4/5 (p1,p2,p3,p4)") } explain { sql """ select * from always_mono_func where to_monday(dt) >= "2019-01-01" """ - contains("partitions=2/5 (p4,p5)") + contains("partitions=3/5 (p1,p4,p5)") } // to_date explain { @@ -203,7 +203,7 @@ suite("always_mono_func") { explain { sql """select * from always_mono_func where date_format(to_monday(dt), 'yyyyMMdd') >= "20190101" """ - contains("partitions=2/5 (p4,p5)") + contains("partitions=3/5 (p1,p4,p5)") } explain { sql """select * from always_mono_func where date_format(last_day(to_monday(dt)), 'yyyyMMdd') < "20190101" """ @@ -216,6 +216,6 @@ suite("always_mono_func") { explain { sql """select * from always_mono_func where date_format(date_trunc(last_day(to_monday(dt)),'month'), 'yyyyMMdd') > "20190101" """ - contains("partitions=2/5 (p4,p5)") + contains("partitions=3/5 (p1,p4,p5)") } } \ No newline at end of file From 342c1f328b32116ddae30f09896df0b33a662d87 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 25 Dec 2024 14:24:56 +0800 Subject: [PATCH 8/8] remove space --- .../trees/expressions/functions/scalar/DateFormat.java | 9 ++++----- .../partition_prune/always_mono_func.groovy | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java index 972c0c8aec709f..997464cf81a5ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java @@ -54,9 +54,9 @@ public class DateFormat extends ScalarFunction FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(DateType.INSTANCE, VarcharType.SYSTEM_DEFAULT) ); - private static final Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-ddHH:mm:ss", - "%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%d%H", "%Y-%m-%d%H:%i", "%Y-%m-%d%H:%i:%s", "%Y-%m-%d%H:%i:%S", - "%Y-%m-%d%T", "%Y%m%d", "%Y%m"); + private static final Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", + "%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%d %H", "%Y-%m-%d %H:%i", "%Y-%m-%d %H:%i:%s", "%Y-%m-%d %H:%i:%S", + "%Y-%m-%d %T", "%Y%m%d", "%Y%m"); /** * constructor with 2 arguments. @@ -90,8 +90,7 @@ public boolean isMonotonic(Literal lower, Literal upper) { if (!(format instanceof VarcharLiteral)) { return false; } - VarcharLiteral varcharLiteral = (VarcharLiteral) format; - String str = varcharLiteral.getValue().replace(" ", ""); + String str = ((VarcharLiteral) format).getValue(); return monoFormat.contains(str); } diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy index 44eb0b873bbfb4..1936dd85accc43 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/always_mono_func.groovy @@ -107,7 +107,7 @@ suite("always_mono_func") { } explain { sql """select * from always_mono_func where date_format(dt, "%Y %m") > "2019 01" """ - contains("partitions=2/5 (p4,p5)") + contains("partitions=5/5 (p1,p2,p3,p4,p5)") } explain {