From 1fe5f2c399985b98547691f37530297d41a58c23 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 11 Dec 2024 21:42:43 +0800 Subject: [PATCH] change to custom rewrite --- .../doris/nereids/jobs/executor/Rewriter.java | 4 +- .../nereids/rules/analysis/CheckAnalysis.java | 31 ------ .../nereids/rules/rewrite/DistinctSplit.java | 98 +++++++++++++------ .../nereids/rules/rewrite/OrExpansion.java | 2 +- .../rules/rewrite/DistinctSplitTest.java | 10 +- .../distinct_split/disitinct_split.out | 56 +++++------ .../distinct_split/disitinct_split.groovy | 72 ++++++++++---- .../aggregate_strategies.groovy | 6 -- 8 files changed, 161 insertions(+), 118 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index dc0b944a3405247..b9749c18f7416e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -443,6 +443,7 @@ public class Rewriter extends AbstractBatchJobExecutor { new CollectCteConsumerOutput() ) ), + // topic("distinct split", topDown(new DistinctSplit())), topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new) ) ) @@ -549,7 +550,8 @@ private static List getWholeTreeRewriteJobs( rewriteJobs.addAll(jobs(topic("or expansion", custom(RuleType.OR_EXPANSION, () -> OrExpansion.INSTANCE)))); } - rewriteJobs.addAll(jobs(topic("distinct split", topDown(new DistinctSplit())))); + rewriteJobs.addAll(jobs(topic("distinct split", + custom(RuleType.DISTINCT_SPLIT, () -> DistinctSplit.INSTANCE)))); if (needSubPathPushDown) { rewriteJobs.addAll(jobs( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index 597cbb68eac9c87..13455720b07a5c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; @@ -139,36 +138,6 @@ private void checkExpressionInputTypes(Plan plan) { } private void checkAggregate(LogicalAggregate aggregate) { - Set aggregateFunctions = aggregate.getAggregateFunctions(); - boolean distinctMultiColumns = false; - for (AggregateFunction func : aggregateFunctions) { - if (!func.isDistinct()) { - continue; - } - if (func.arity() <= 1) { - continue; - } - for (int i = 1; i < func.arity(); i++) { - if (!func.child(i).getInputSlots().isEmpty() && !(func.child(i) instanceof OrderExpression)) { - // think about group_concat(distinct col_1, ',') - distinctMultiColumns = true; - break; - } - } - if (distinctMultiColumns) { - break; - } - } - - long distinctFunctionNum = 0; - for (AggregateFunction aggregateFunction : aggregateFunctions) { - distinctFunctionNum += aggregateFunction.isDistinct() ? 1 : 0; - } - - // if (distinctMultiColumns && distinctFunctionNum > 1) { - // throw new AnalysisException( - // "The query contains multi count distinct or sum distinct, each can't have multi columns"); - // } for (Expression expr : aggregate.getGroupByExpressions()) { if (expr.anyMatch(AggregateFunction.class::isInstance)) { throw new AnalysisException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctSplit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctSplit.java index d8c9de81c4251c0..cf557245d2ae305 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctSplit.java @@ -18,8 +18,9 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.rules.rewrite.DistinctSplit.DistinctSplitContext; import org.apache.doris.nereids.trees.copier.DeepCopierContext; import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; import org.apache.doris.nereids.trees.expressions.Alias; @@ -38,8 +39,12 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.ImmutableList; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -62,40 +67,61 @@ * +--LogicalAggregate(output:count(distinct b)) * +--LogicalCTEConsumer * */ -public class DistinctSplit extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalAggregate() - // TODO with source repeat aggregate need to be supported in future - .whenNot(agg -> agg.getSourceRepeat().isPresent()) - .thenApply(ctx -> doSplit(ctx.root, ctx.cascadesContext)) - .toRule(RuleType.DISTINCT_SPLIT); +public class DistinctSplit extends DefaultPlanRewriter implements CustomRewriter { + public static DistinctSplit INSTANCE = new DistinctSplit(); + + /**DistinctSplitContext*/ + public static class DistinctSplitContext { + List> cteProducerList; + StatementContext statementContext; + CascadesContext cascadesContext; + + public DistinctSplitContext(StatementContext statementContext, CascadesContext cascadesContext) { + this.statementContext = statementContext; + this.cteProducerList = new ArrayList<>(); + this.cascadesContext = cascadesContext; + } } - private static boolean isDistinctMultiColumns(AggregateFunction func) { - if (func.arity() <= 1) { - return false; + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + DistinctSplitContext ctx = new DistinctSplitContext( + jobContext.getCascadesContext().getStatementContext(), jobContext.getCascadesContext()); + plan = plan.accept(this, ctx); + for (int i = ctx.cteProducerList.size() - 1; i >= 0; i--) { + LogicalCTEProducer producer = ctx.cteProducerList.get(i); + plan = new LogicalCTEAnchor<>(producer.getCteId(), producer, plan); } - for (int i = 1; i < func.arity(); ++i) { - // think about group_concat(distinct col_1, ',') - if (!(func.child(i) instanceof OrderExpression) && !func.child(i).getInputSlots().isEmpty()) { - return true; - } + return plan; + } + + @Override + public Plan visitLogicalCTEAnchor( + LogicalCTEAnchor anchor, DistinctSplitContext ctx) { + Plan child1 = anchor.child(0).accept(this, ctx); + DistinctSplitContext consumerContext = + new DistinctSplitContext(ctx.statementContext, ctx.cascadesContext); + Plan child2 = anchor.child(1).accept(this, consumerContext); + for (int i = consumerContext.cteProducerList.size() - 1; i >= 0; i--) { + LogicalCTEProducer producer = consumerContext.cteProducerList.get(i); + child2 = new LogicalCTEAnchor<>(producer.getCteId(), producer, child2); } - return false; + return anchor.withChildren(ImmutableList.of(child1, child2)); } - private static Plan doSplit(LogicalAggregate agg, CascadesContext ctx) { + @Override + public Plan visitLogicalAggregate(LogicalAggregate agg, DistinctSplitContext ctx) { List distinctFuncWithAlias = new ArrayList<>(); List otherAggFuncs = new ArrayList<>(); - if (!needTransform(agg, distinctFuncWithAlias, otherAggFuncs)) { - return null; + if (!needTransform((LogicalAggregate) agg, distinctFuncWithAlias, otherAggFuncs)) { + return agg; } LogicalAggregate cloneAgg = (LogicalAggregate) LogicalPlanDeepCopier.INSTANCE .deepCopy(agg, new DeepCopierContext()); - LogicalCTEProducer producer = new LogicalCTEProducer<>(ctx.getStatementContext().getNextCTEId(), + LogicalCTEProducer producer = new LogicalCTEProducer<>(ctx.statementContext.getNextCTEId(), cloneAgg.child()); + ctx.cteProducerList.add(producer); Map originToProducerSlot = new HashMap<>(); for (int i = 0; i < agg.child().getOutput().size(); ++i) { Slot originSlot = agg.child().getOutput().get(i); @@ -106,14 +132,14 @@ private static Plan doSplit(LogicalAggregate agg, CascadesContext ctx) { otherAggFuncs = ExpressionUtils.replace((List) otherAggFuncs, originToProducerSlot); // construct cte consumer and aggregate List> newAggs = new ArrayList<>(); - // All aggFunc except count distinct are placed in the first one + // All otherAggFuncs are placed in the first one Map newToOriginDistinctFuncAlias = new HashMap<>(); List outputJoinGroupBys = new ArrayList<>(); for (int i = 0; i < distinctFuncWithAlias.size(); ++i) { Expression distinctAggFunc = distinctFuncWithAlias.get(i).child(0); - LogicalCTEConsumer consumer = new LogicalCTEConsumer(ctx.getStatementContext().getNextRelationId(), + LogicalCTEConsumer consumer = new LogicalCTEConsumer(ctx.statementContext.getNextRelationId(), producer.getCteId(), "", producer); - ctx.putCTEIdToConsumer(consumer); + ctx.cascadesContext.putCTEIdToConsumer(consumer); Map producerToConsumerSlotMap = new HashMap<>(); for (Map.Entry entry : consumer.getConsumerToProducerOutputMap().entrySet()) { producerToConsumerSlotMap.put(entry.getValue(), entry.getKey()); @@ -143,12 +169,28 @@ private static Plan doSplit(LogicalAggregate agg, CascadesContext ctx) { } List groupBy = agg.getGroupByExpressions(); LogicalJoin join = constructJoin(newAggs, groupBy); - LogicalProject project = constructProject(groupBy, newToOriginDistinctFuncAlias, + return constructProject(groupBy, newToOriginDistinctFuncAlias, outputJoinGroupBys, join); - return new LogicalCTEAnchor(producer.getCteId(), producer, project); + } + + private static boolean isDistinctMultiColumns(AggregateFunction func) { + if (func.arity() <= 1) { + return false; + } + for (int i = 1; i < func.arity(); ++i) { + // think about group_concat(distinct col_1, ',') + if (!(func.child(i) instanceof OrderExpression) && !func.child(i).getInputSlots().isEmpty()) { + return true; + } + } + return false; } private static boolean needTransform(LogicalAggregate agg, List aliases, List otherAggFuncs) { + // TODO with source repeat aggregate need to be supported in future + if (agg.getSourceRepeat().isPresent()) { + return false; + } Set distinctFunc = new HashSet<>(); boolean distinctMultiColumns = false; for (NamedExpression namedExpression : agg.getOutputExpressions()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java index 369a210eb2f99c1..378ee0f67aef201 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java @@ -79,7 +79,7 @@ public class OrExpansion extends DefaultPlanRewriter implem @Override public Plan rewriteRoot(Plan plan, JobContext jobContext) { - OrExpandsionContext ctx = new OrExpandsionContext( + OrExpandsionContext ctx = new OrExpandsionContext( jobContext.getCascadesContext().getStatementContext(), jobContext.getCascadesContext()); plan = plan.accept(this, ctx); for (int i = ctx.cteProducerList.size() - 1; i >= 0; i--) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctSplitTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctSplitTest.java index 948dc6eab7bf2f2..f132afc5f4d81fc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctSplitTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctSplitTest.java @@ -40,7 +40,7 @@ void multiCountWithoutGby() { String sql = "select count(distinct b), count(distinct a) from test_distinct_multi"; PlanChecker.from(connectContext).checkExplain(sql, planner -> { Plan plan = planner.getOptimizedPlan(); - MatchingUtils.assertMatches(plan, physicalResultSink(physicalCTEAnchor(physicalCTEProducer(any()), physicalProject(physicalNestedLoopJoin( + MatchingUtils.assertMatches(plan, physicalCTEAnchor(physicalCTEProducer(any()), physicalResultSink(physicalProject(physicalNestedLoopJoin( physicalHashAggregate(physicalDistribute(physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any())))))), physicalDistribute(physicalHashAggregate(physicalDistribute(physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any()))))))) ))))); @@ -52,7 +52,7 @@ void multiSumWithoutGby() { String sql = "select sum(distinct b), sum(distinct a) from test_distinct_multi"; PlanChecker.from(connectContext).checkExplain(sql, planner -> { Plan plan = planner.getOptimizedPlan(); - MatchingUtils.assertMatches(plan, physicalResultSink(physicalCTEAnchor(physicalCTEProducer(any()), physicalProject(physicalNestedLoopJoin( + MatchingUtils.assertMatches(plan, physicalCTEAnchor(physicalCTEProducer(any()), physicalResultSink(physicalProject(physicalNestedLoopJoin( physicalHashAggregate(physicalDistribute(physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any())))))), physicalDistribute(physicalHashAggregate(physicalDistribute(physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any()))))))) ))))); @@ -64,7 +64,7 @@ void SumCountWithoutGby() { String sql = "select sum(distinct b), count(distinct a) from test_distinct_multi"; PlanChecker.from(connectContext).checkExplain(sql, planner -> { Plan plan = planner.getOptimizedPlan(); - MatchingUtils.assertMatches(plan, physicalResultSink(physicalCTEAnchor(physicalCTEProducer(any()), physicalProject(physicalNestedLoopJoin( + MatchingUtils.assertMatches(plan, physicalCTEAnchor(physicalCTEProducer(any()), physicalResultSink(physicalProject(physicalNestedLoopJoin( physicalHashAggregate(physicalDistribute(physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any())))))), physicalDistribute(physicalHashAggregate(physicalDistribute(physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any()))))))) ))))); @@ -76,7 +76,7 @@ void CountMultiColumnsWithoutGby() { String sql = "select count(distinct b,c), count(distinct a,b) from test_distinct_multi"; PlanChecker.from(connectContext).checkExplain(sql, planner -> { Plan plan = planner.getOptimizedPlan(); - MatchingUtils.assertMatches(plan, physicalResultSink(physicalCTEAnchor(physicalCTEProducer(any()), physicalProject(physicalNestedLoopJoin( + MatchingUtils.assertMatches(plan, physicalCTEAnchor(physicalCTEProducer(any()), physicalResultSink(physicalProject(physicalNestedLoopJoin( physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any())))), physicalDistribute(physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any()))))) ))))); @@ -88,7 +88,7 @@ void CountMultiColumnsWithGby() { String sql = "select count(distinct b,c), count(distinct a,b) from test_distinct_multi group by d"; PlanChecker.from(connectContext).checkExplain(sql, planner -> { Plan plan = planner.getOptimizedPlan(); - MatchingUtils.assertMatches(plan, physicalResultSink(physicalCTEAnchor(physicalCTEProducer(any()), physicalDistribute(physicalProject(physicalHashJoin( + MatchingUtils.assertMatches(plan, physicalCTEAnchor(physicalCTEProducer(any()), physicalResultSink(physicalDistribute(physicalProject(physicalHashJoin( physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any())))), physicalHashAggregate(physicalHashAggregate(physicalDistribute(physicalHashAggregate(any())))) )))))); diff --git a/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out b/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out index 9828a42875c2536..53e344b40d45113 100644 --- a/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out +++ b/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out @@ -255,10 +255,10 @@ 1 4 25 2 -- !multi_count_without_gby -- -PhysicalResultSink ---PhysicalCteAnchor ( cteId=CTEId#0 ) -----PhysicalCteProducer ( cteId=CTEId#0 ) -------PhysicalOlapScan[test_distinct_multi] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOlapScan[test_distinct_multi] +--PhysicalResultSink ----NestedLoopJoin[CROSS_JOIN] ------hashAgg[DISTINCT_GLOBAL] --------hashAgg[DISTINCT_LOCAL] @@ -272,10 +272,10 @@ PhysicalResultSink --------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !multi_sum_without_gby -- -PhysicalResultSink ---PhysicalCteAnchor ( cteId=CTEId#0 ) -----PhysicalCteProducer ( cteId=CTEId#0 ) -------PhysicalOlapScan[test_distinct_multi] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOlapScan[test_distinct_multi] +--PhysicalResultSink ----NestedLoopJoin[CROSS_JOIN] ------hashAgg[DISTINCT_GLOBAL] --------hashAgg[DISTINCT_LOCAL] @@ -289,10 +289,10 @@ PhysicalResultSink --------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !sum_count_without_gby -- -PhysicalResultSink ---PhysicalCteAnchor ( cteId=CTEId#0 ) -----PhysicalCteProducer ( cteId=CTEId#0 ) -------PhysicalOlapScan[test_distinct_multi] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOlapScan[test_distinct_multi] +--PhysicalResultSink ----NestedLoopJoin[CROSS_JOIN] ------hashAgg[DISTINCT_GLOBAL] --------hashAgg[DISTINCT_LOCAL] @@ -306,10 +306,10 @@ PhysicalResultSink --------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !multi_count_mulitcols_without_gby -- -PhysicalResultSink ---PhysicalCteAnchor ( cteId=CTEId#0 ) -----PhysicalCteProducer ( cteId=CTEId#0 ) -------PhysicalOlapScan[test_distinct_multi] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOlapScan[test_distinct_multi] +--PhysicalResultSink ----NestedLoopJoin[CROSS_JOIN] ------hashAgg[DISTINCT_LOCAL] --------hashAgg[GLOBAL] @@ -321,10 +321,10 @@ PhysicalResultSink ------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !multi_count_mulitcols_with_gby -- -PhysicalResultSink ---PhysicalCteAnchor ( cteId=CTEId#0 ) -----PhysicalCteProducer ( cteId=CTEId#0 ) -------PhysicalOlapScan[test_distinct_multi] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOlapScan[test_distinct_multi] +--PhysicalResultSink ----hashJoin[INNER_JOIN] hashCondition=((.d = .d)) otherCondition=() ------hashAgg[DISTINCT_LOCAL] --------hashAgg[GLOBAL] @@ -336,10 +336,10 @@ PhysicalResultSink ------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !three_count_mulitcols_without_gby -- -PhysicalResultSink ---PhysicalCteAnchor ( cteId=CTEId#0 ) -----PhysicalCteProducer ( cteId=CTEId#0 ) -------PhysicalOlapScan[test_distinct_multi] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOlapScan[test_distinct_multi] +--PhysicalResultSink ----NestedLoopJoin[CROSS_JOIN] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[DISTINCT_LOCAL] @@ -356,10 +356,10 @@ PhysicalResultSink ------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !four_count_mulitcols_with_gby -- -PhysicalResultSink ---PhysicalCteAnchor ( cteId=CTEId#0 ) -----PhysicalCteProducer ( cteId=CTEId#0 ) -------PhysicalOlapScan[test_distinct_multi] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOlapScan[test_distinct_multi] +--PhysicalResultSink ----hashJoin[INNER_JOIN] hashCondition=((.d = .d)) otherCondition=() ------hashJoin[INNER_JOIN] hashCondition=((.d = .d)) otherCondition=() --------hashAgg[DISTINCT_LOCAL] diff --git a/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy b/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy index d90a8a8bf15d130..eec9609e12facc6 100644 --- a/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy +++ b/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy @@ -16,46 +16,48 @@ // under the License. suite("distinct_split") { - // 需要多一点测试用例 sql "drop table if exists test_distinct_multi" sql "create table test_distinct_multi(a int, b int, c int, d varchar(10), e date) distributed by hash(a) properties('replication_num'='1');" sql "insert into test_distinct_multi values(1,2,3,'abc','2024-01-02'),(1,2,4,'abc','2024-01-03'),(2,2,4,'abcd','2024-01-02'),(1,2,3,'abcd','2024-01-04'),(1,2,4,'eee','2024-02-02'),(2,2,4,'abc','2024-01-02');" - //这个的维度目前只有count *4 加入四个函数进行测试 - //000 distinct有1列，无group by，投影列有1个count(distinct) 四阶段agg + + // first bit 0 means distinct 1 col, 1 means distinct more than 1 col; second bit 0 means without group by, 1 means with group by; + // third bit 0 means there is 1 count(distinct) in projects, 1 means more than 1 count(distinct) in projects. + + //000 distinct has 1 column, no group by, projection column has 1 count (distinct). four stages agg qt_000_count """select count(distinct a) from test_distinct_multi""" - //001 distinct有1列，无group by，投影列有多个count(distinct) 两阶段agg，第二阶段单点计算很慢 + //001 distinct has 1 column, no group by, and multiple counts (distinct) in the projection column. The two-stage agg is slow for single point calculation in the second stage qt_001_count """select count(distinct b), count(distinct a) from test_distinct_multi""" - //010 distinct有1列，有group by，投影列有1个count(distinct) 两阶段agg，第二阶段按照group by hash + //010 distinct has 1 column with group by, and the projection column has 1 count (distinct). two-stage agg. The second stage follows group by hash qt_010_count """select count(distinct a) from test_distinct_multi group by b order by 1""" qt_010_count_same_column_with_groupby """select count(distinct a) from test_distinct_multi group by a order by 1""" - //011 distinct有1列，有group by，投影列有多个count(distinct) 两阶段agg，第二阶段按照group by hash + //011 distinct has one column with group by, and the projection column has multiple counts (distinct). two stages agg. The second stage follows group by hash qt_011_count_same_column_with_groupby """select count(distinct a),count(distinct b) from test_distinct_multi group by a order by 1,2""" qt_011_count_diff_column_with_groupby """select count(distinct a),count(distinct b) from test_distinct_multi group by c order by 1,2""" qt_011_count_diff_column_with_groupby_multi """select count(distinct a),count(distinct b) from test_distinct_multi group by a,c order by 1,2""" qt_011_count_diff_column_with_groupby_all """select count(distinct a),count(distinct b) from test_distinct_multi group by a,b,c order by 1,2""" - //100 distinct有多列，无group by，投影列有1个count(distinct) 三阶段agg，第二阶段gather + //100 distinct columns with no group by, projection column with 1 count (distinct). Three stage agg, second stage gather qt_100 """select count(distinct a,b) from test_distinct_multi""" - //101 distinct有多列，无group by，投影列有多个count(distinct) （拦截）如果去掉拦截，可以执行，但是结果错误 + //101 distinct has multiple columns, no group by, and multiple counts (distinct) in the projection column (intercept). If the intercept is removed, it can be executed, but the result is incorrect qt_101 """select count(distinct a,b), count(distinct a,c) from test_distinct_multi""" qt_101_count_one_col_and_two_col """select count(distinct a,b), count(distinct c) from test_distinct_multi""" qt_101_count_one_col_and_two_col """select count(distinct a,b), count(distinct a) from test_distinct_multi""" - //110 distinct有多列，有group by，投影列有1个count(distinct) 三阶段agg，第二阶段按照group by hash + //110 distinct has multiple columns, including group by, and the projection column has one count (distinct). three-stage agg. The second stage follows group by hash qt_110_count_diff_column_with_groupby """select count(distinct a,b) from test_distinct_multi group by c order by 1""" qt_110_count_same_column_with_groupby1 """select count(distinct a,b) from test_distinct_multi group by a order by 1""" qt_110_count_same_column_with_groupby2 """select count(distinct a,b) from test_distinct_multi group by a,b order by 1""" - //111 distinct有多列，有group by，投影列有多个count(distinct) （拦截）如果去掉拦截，可以执行，但是结果错误 + //111 distinct has multiple columns, including group by, and the projection column has multiple counts (distinct) (intercept). If the intercept is removed, it can be executed, but the result is incorrect qt_111_count_same_column_with_groupby1 """select count(distinct a,b), count(distinct a,c) from test_distinct_multi group by c order by 1,2""" qt_111_count_same_column_with_groupby2 """select count(distinct a,b), count(distinct c) from test_distinct_multi group by a,c order by 1,2""" qt_111_count_diff_column_with_groupby """select count(distinct a,b), count(distinct a) from test_distinct_multi group by c order by 1,2""" - //测试一下有别的函数的时候 + // testing other functions qt_000_count_other_func """select count(distinct a), max(b),sum(c),min(a) from test_distinct_multi""" qt_001_count_other_func """select count(distinct b), count(distinct a), max(b),sum(c),min(a) from test_distinct_multi""" qt_010_count_other_func """select count(distinct a), max(b),sum(c),min(a),b from test_distinct_multi group by b order by 1,2,3,4,5""" @@ -88,32 +90,31 @@ suite("distinct_split") { qt_111_four_gby_multi """select count(distinct a,b), count(distinct a,c) , count(distinct a), count(distinct c) from test_distinct_multi group by e,a,b order by 1,2,3,4""" qt_111_five_gby_multi """select count(distinct a,b), count(distinct a,c) , count(distinct a,d), count(distinct c) , count(distinct a,b,c,d) from test_distinct_multi group by e,a,b,c,d order by 1,2,3,4,5""" - - // sum 有两个维度 1.投影列一个还是多个(0为1个，1为多个) 2，有无group by（0为无，1为有） + // sum has two dimensions: 1. Is there one or more projection columns (0 for one, 1 for more) 2. Is there a group by (0 for none, 1 for yes) qt_00_sum """select sum(distinct b) from test_distinct_multi""" qt_10_sum """select sum(distinct b), sum(distinct a) from test_distinct_multi""" qt_01_sum """select sum(distinct b) from test_distinct_multi group by a order by 1""" qt_11_sum """select sum(distinct b), sum(distinct a) from test_distinct_multi group by a order by 1,2""" - // avg 有两个维度 1.投影列一个还是多个(0为1个，1为多个) 2，有无group by（0为无，1为有） + // avg has two dimensions: 1. Is there one or more projection columns (0 for one, 1 for more) 2. Is there a group by (0 for no, 1 for yes) qt_00_avg """select avg(distinct b) from test_distinct_multi""" qt_10_avg """select avg(distinct b), avg(distinct a) from test_distinct_multi""" qt_01_avg """select avg(distinct b) from test_distinct_multi group by a order by 1""" qt_11_avg """select avg(distinct b), avg(distinct a) from test_distinct_multi group by a order by 1,2""" - //group_concat 好像有order by distinct没有支持 + //group_concat sql """select group_concat(distinct d order by d) from test_distinct_multi""" sql """select group_concat(distinct d order by d), group_concat(distinct cast(a as string) order by cast(a as string)) from test_distinct_multi""" sql """select group_concat(distinct d order by d) from test_distinct_multi group by a order by 1""" sql """select group_concat(distinct d order by d), group_concat(distinct cast(a as string) order by cast(a as string)) from test_distinct_multi group by a order by 1,2""" - //混合 + // mixed distinct function qt_count_sum_avg_no_gby "select sum(distinct b), count(distinct a), avg(distinct c) from test_distinct_multi" qt_count_multi_sum_avg_no_gby "select sum(distinct b), count(distinct a,d), avg(distinct c) from test_distinct_multi" qt_count_sum_avg_with_gby "select sum(distinct b), count(distinct a), avg(distinct c) from test_distinct_multi group by b,a order by 1,2,3" qt_count_multi_sum_avg_with_gby "select sum(distinct b), count(distinct a,d), avg(distinct c) from test_distinct_multi group by a,b order by 1,2,3" - // 上层有引用查询 + // There is a reference query in the upper layer qt_multi_sum_has_upper """select c1+ c2 from (select sum(distinct b) c1, sum(distinct a) c2 from test_distinct_multi) t""" qt_000_count_has_upper """select abs(c1) from (select count(distinct a) c1 from test_distinct_multi) t""" qt_010_count_has_upper """select c1+100 from (select count(distinct a) c1 from test_distinct_multi group by b) t order by 1""" @@ -124,7 +125,42 @@ suite("distinct_split") { qt_010_count_sum_other_func_has_upper """select sum(c0),max(c1+c2), min(c2+c3+c4),max(b) from (select sum(distinct b) c0,count(distinct a) c1, max(b) c2,sum(c) c3,min(a) c4,b from test_distinct_multi group by b) t""" qt_010_count_other_func_has_upper"""select sum(c0), max(c1+c2), min(c2+c3+c4),max(b) from (select count(distinct b) c0,count(distinct a) c1, max(b) c2,sum(c) c3,min(a) c4,b from test_distinct_multi group by b) t""" - // 在cte里面或者在嵌套cte里面。 + // In cte or in nested cte. + qt_cte_producer """with t1 as (select a,b from test_distinct_multi) + select count(distinct t.a), count(distinct tt.b) from t1 t cross join t1 tt;""" + qt_cte_consumer """with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi) + select * from t1 t cross join t1 tt;""" + qt_cte_multi_producer """ + with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi), + t2 as (select sum(distinct a), sum(distinct b) from test_distinct_multi), + t3 as (select sum(distinct a), count(distinct b) from test_distinct_multi) + select * from t1,t2,t3; + """ + qt_multi_cte_nest """ + with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi), + t2 as (select sum(distinct a), sum(distinct b) from test_distinct_multi), + t3 as (select sum(distinct a), count(distinct b) from test_distinct_multi) + select * from t1,t2,t3, (with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi), + t2 as (select sum(distinct a), sum(distinct b) from test_distinct_multi), + t3 as (select sum(distinct a), count(distinct b) from test_distinct_multi) + select * from t1,t2,t3) tmp; + """ + qt_multi_cte_nest2 """ + with t1 as (with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi), + t2 as (select sum(distinct a), sum(distinct b) from test_distinct_multi), + t3 as (select sum(distinct a), count(distinct b) from test_distinct_multi) + select * from t1,t2,t3, (with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi), + t2 as (select sum(distinct a), sum(distinct b) from test_distinct_multi), + t3 as (select sum(distinct a), count(distinct b) from test_distinct_multi) + select * from t1,t2,t3) tmp) + select * from t1,t1,(with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi), + t2 as (select sum(distinct a), sum(distinct b) from test_distinct_multi), + t3 as (select sum(distinct a), count(distinct b) from test_distinct_multi) + select * from t1,t2,t3, (with t1 as (select count(distinct a), count(distinct b) from test_distinct_multi), + t2 as (select sum(distinct a), sum(distinct b) from test_distinct_multi), + t3 as (select sum(distinct a), count(distinct b) from test_distinct_multi) + select * from t1,t2,t3) tmp) t + """ // shape sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" diff --git a/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy b/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy index 1b546db0ff8eae2..aeb39fb275af3d8 100644 --- a/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy +++ b/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy @@ -149,12 +149,6 @@ suite("aggregate_strategies") { from $tableName )a group by c""" - - - test { - sql "select count(distinct id, name), count(distinct id) from $tableName" - exception "The query contains multi count distinct or sum distinct, each can't have multi columns" - } } test_aggregate_strategies('test_bucket1_table', 1)