Skip to content

Commit

Permalink
add check limit
Browse files Browse the repository at this point in the history
  • Loading branch information
@yujun777
yujun777 committed Nov 30, 2024
1 parent 7ed708d commit 7b19651
Show file tree
Hide file tree
Showing 7 changed files with 449 additions and 79 deletions.
184 changes: 130 additions & 54 deletions ...ain/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,18 @@

package org.apache.doris.nereids.rules.expression.rules;

import org.apache.doris.common.Pair;
import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule;
import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
Expand All @@ -45,16 +44,16 @@
import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.NumericLiteral;
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.TypeCoercionUtils;

import com.google.common.base.Preconditions;
Expand All @@ -63,9 +62,10 @@
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.List;
import java.util.Optional;

/**
* simplify comparison
* simplify comparison, not support large int.
* such as: cast(c1 as DateV2) >= DateV2Literal --> c1 >= DateLiteral
* cast(c1 AS double) > 2.0 --> c1 >= 2 (c1 is integer like type)
*/
Expand Down Expand Up @@ -100,22 +100,25 @@ public static Expression simplify(ComparisonPredicate cp) {
Expression left = cp.left();
Expression right = cp.right();

// float like type: float, double
if (left.getDataType().isFloatLikeType() && right.getDataType().isFloatLikeType()) {
return processFloatLikeTypeCoercion(cp, left, right);
}
Expression result;

// decimalv3 type
if (left.getDataType() instanceof DecimalV3Type && right.getDataType() instanceof DecimalV3Type) {
return processDecimalV3TypeCoercion(cp, left, right);
// process type coercion
if (left.getDataType().isFloatLikeType() && right.getDataType().isFloatLikeType()) {
result = processFloatLikeTypeCoercion(cp, left, right);
} else if (left.getDataType() instanceof DecimalV3Type && right.getDataType() instanceof DecimalV3Type) {
result = processDecimalV3TypeCoercion(cp, left, right);
} else if (left.getDataType() instanceof DateLikeType && right.getDataType() instanceof DateLikeType) {
result = processDateLikeTypeCoercion(cp, left, right);
} else {
result = cp;
}

// date like type
if (left.getDataType() instanceof DateLikeType && right.getDataType() instanceof DateLikeType) {
return processDateLikeTypeCoercion(cp, left, right);
if (result instanceof ComparisonPredicate && ((ComparisonPredicate) result).right() instanceof NumericLiteral) {
ComparisonPredicate cmp = (ComparisonPredicate) result;
result = processTypeRangeLimitComparison(cmp, cmp.left(), (NumericLiteral) cmp.right());
}

return cp;
return result;
}

private static Expression processComparisonPredicateDateTimeV2Literal(
Expand All @@ -130,17 +133,13 @@ private static Expression processComparisonPredicateDateTimeV2Literal(
if (right.getMicroSecond() == originValue) {
return comparisonPredicate.withChildren(left, right);
} else {
if (left.nullable()) {
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return new And(new IsNull(left), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.of(false);
}
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return ExpressionUtils.falseOrNull(left);
}
} else if (comparisonPredicate instanceof NullSafeEqual) {
long originValue = right.getMicroSecond();
Expand Down Expand Up @@ -241,18 +240,13 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa
comparisonPredicate.withChildren(left, new DecimalV3Literal(
literal.getValue().setScale(toScale, RoundingMode.UNNECESSARY))));
} catch (ArithmeticException e) {
if (left.nullable()) {
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return new And(new IsNull(left),
new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.of(false);
}
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return ExpressionUtils.falseOrNull(left);
}
} else if (comparisonPredicate instanceof NullSafeEqual) {
try {
Expand Down Expand Up @@ -283,21 +277,18 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa
private static Expression processIntegerDecimalLiteralComparison(
ComparisonPredicate comparisonPredicate, Expression left, BigDecimal literal) {
// we only process isIntegerLikeType, which are tinyint, smallint, int, bigint
if (literal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0) {
if (literal.compareTo(new BigDecimal(Long.MIN_VALUE)) >= 0
&& literal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0) {
literal = literal.stripTrailingZeros();
if (literal.scale() > 0) {
if (comparisonPredicate instanceof EqualTo) {
if (left.nullable()) {
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return new And(new IsNull(left), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.of(false);
}
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return ExpressionUtils.falseOrNull(left);
} else if (comparisonPredicate instanceof NullSafeEqual) {
return BooleanLiteral.of(false);
} else if (comparisonPredicate instanceof GreaterThan
Expand All @@ -322,10 +313,95 @@ private static Expression processIntegerDecimalLiteralComparison(
return comparisonPredicate;
}

private static Expression processTypeRangeLimitComparison(ComparisonPredicate cp, Expression left,
NumericLiteral right) {
BigDecimal typeMinValue = null;
BigDecimal typeMaxValue = null;
// cmp float like have lost precision, for example float.max_value + 0.01 still eval to float.max_value
if (left.getDataType().isIntegerLikeType() || left.getDataType().isDecimalV3Type()) {
Optional<Pair<BigDecimal, BigDecimal>> minMaxOpt =
TypeCoercionUtils.getDataTypeMinMaxValue(left.getDataType());
if (minMaxOpt.isPresent()) {
typeMinValue = minMaxOpt.get().first;
typeMaxValue = minMaxOpt.get().second;
}
}

// cast(child as dataType2) range should be:
// [ max(childDataType.min_value, dataType2.min_value), min(childDataType.max_value, dataType2.max_value)]
if (left instanceof Cast) {
left = ((Cast) left).child();
if (left.getDataType().isIntegerLikeType() || left.getDataType().isDecimalV3Type()) {
Optional<Pair<BigDecimal, BigDecimal>> minMaxOpt =
TypeCoercionUtils.getDataTypeMinMaxValue(left.getDataType());
if (minMaxOpt.isPresent()) {
if (typeMinValue == null || typeMinValue.compareTo(minMaxOpt.get().first) < 0) {
typeMinValue = minMaxOpt.get().first;
}
if (typeMaxValue == null || typeMaxValue.compareTo(minMaxOpt.get().second) > 0) {
typeMaxValue = minMaxOpt.get().second;
}
}
}
}

if (typeMinValue == null || typeMaxValue == null) {
return cp;
}
BigDecimal literal = new BigDecimal(right.getStringValue());
int cmpMin = literal.compareTo(typeMinValue);
int cmpMax = literal.compareTo(typeMaxValue);
if (cp instanceof EqualTo) {
if (cmpMin < 0 || cmpMax > 0) {
return ExpressionUtils.falseOrNull(left);
}
} else if (cp instanceof NullSafeEqual) {
if (cmpMin < 0 || cmpMax > 0) {
return BooleanLiteral.of(false);
}
} else if (cp instanceof GreaterThan) {
if (cmpMin < 0) {
return ExpressionUtils.trueOrNull(left);
}
if (cmpMax >= 0) {
return ExpressionUtils.falseOrNull(left);
}
} else if (cp instanceof GreaterThanEqual) {
if (cmpMin <= 0) {
return ExpressionUtils.trueOrNull(left);
}
if (cmpMax == 0) {
return new EqualTo(cp.left(), cp.right());
}
if (cmpMax > 0) {
return ExpressionUtils.falseOrNull(left);
}
} else if (cp instanceof LessThan) {
if (cmpMin <= 0) {
return ExpressionUtils.falseOrNull(left);
}
if (cmpMax > 0) {
return ExpressionUtils.trueOrNull(left);
}
} else if (cp instanceof LessThanEqual) {
if (cmpMin < 0) {
return ExpressionUtils.falseOrNull(left);
}
if (cmpMin == 0) {
return new EqualTo(cp.left(), cp.right());
}
if (cmpMax >= 0) {
return ExpressionUtils.trueOrNull(left);
}
}
return cp;
}

private static IntegerLikeLiteral convertDecimalToIntegerLikeLiteral(BigDecimal decimal) {
Preconditions.checkArgument(
decimal.scale() <= 0 && decimal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0,
"decimal literal must have 0 scale and smaller than Long.MAX_VALUE");
Preconditions.checkArgument(decimal.scale() <= 0
&& decimal.compareTo(new BigDecimal(Long.MIN_VALUE)) >= 0
&& decimal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0,
"decimal literal must have 0 scale and in range [Long.MIN_VALUE, Long.MAX_VALUE]");
long val = decimal.longValue();
if (val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE) {
return new TinyIntLiteral((byte) val);
Expand Down
17 changes: 2 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,11 @@
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.collect.BoundType;
Expand Down Expand Up @@ -287,11 +282,7 @@ public ValueDesc intersect(ValueDesc other) {

@Override
public Expression toExpression() {
if (reference.nullable()) {
return new And(new IsNull(reference), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.FALSE;
}
return ExpressionUtils.falseOrNull(reference);
}
}

Expand Down Expand Up @@ -378,11 +369,7 @@ public Expression toExpression() {
if (!result.isEmpty()) {
return ExpressionUtils.and(result);
} else {
if (reference.nullable()) {
return new Or(new Not(new IsNull(reference)), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.TRUE;
}
return ExpressionUtils.trueOrNull(reference);
}
}

Expand Down
16 changes: 16 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,22 @@ public static Expression or(Collection<Expression> expressions) {
return combineAsLeftDeepTree(Or.class, expressions);
}

public static Expression falseOrNull(Expression expression) {
if (expression.nullable()) {
return new And(new IsNull(expression), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.FALSE;
}
}

public static Expression trueOrNull(Expression expression) {
if (expression.nullable()) {
return new Or(new Not(new IsNull(expression)), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.TRUE;
}
}

/**
* Use AND/OR to combine expressions together.
*/
Expand Down
Loading

0 comments on commit 7b19651

Please sign in to comment.