From 81f3c4841abfc1d69b0fbf96c6521e38b9777085 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 23 Dec 2024 17:50:07 +0800 Subject: [PATCH] [enhancement](nereids) improve lots of values in `insert into values` statement (#40202) improve lots of values in `insert into values` statement by bypass NereidsPlanner the main logic is 1. `InsertUtils.normalizePlan` use `FoldConstantRuleOnFE` to reduce the expression, e.g. `values(date(now())` 2. `FastInsertIntoValuesPlanner` skip most of rules to analyze and rewrite `LogicalInlineTable` to `LogicalUnion` or `LogicalOneRowRelation` 3. fast parse date time string without date format 4. getHintMap and normal lexer share the same tokens 5. `set enable_fast_analyze_into_values=false` can force to execute all optimize rules, when we meet some bugs in `FastInsertIntoValuesPlanner` test: insert 1000 rows with 1000 columns, the columns contains int, bigint, decimal(26,7), date, datetime, varchar(10 chinese chars) +---------------------------------+------------------------------------------------------+--------------------------+--------------------------+ |FastInsertIntoValuesPlanner |NereidsPlanner(enable_fast_analyze_into_values=false) |Legacy optimizer in 2.1.6 | Nereids planner in 2.1.6 | +---------------------------------+------------------------------------------------------+--------------------------+--------------------------+ |16s(bottleneck is antlr's lexer) |32s |16s |80s | +---------------------------------+------------------------------------------------------+--------------------------+--------------------------+ If you use FastInsertIntoValuesPlanner with group commit in a transaction, the time can reduce to 12s. TODO: build a custom lexer. in my hand write lexer test, FastInsertIntoValuesPlanner without group commit can reduce 16s to 12s, but it will take more effort: RegularExpression -> NFA -> DFA -> minimal DFA -> Lexer codegen --- be/src/http/http_channel.cpp | 3 +- .../org/apache/doris/nereids/DorisParser.g4 | 6 +- .../apache/doris/nereids/CascadesContext.java | 9 +- .../apache/doris/nereids/NereidsPlanner.java | 19 +- .../nereids/analyzer/UnboundInlineTable.java | 87 ++++++++ .../doris/nereids/analyzer/UnboundPlan.java | 39 ++++ .../nereids/analyzer/UnboundTableSink.java | 9 + .../translator/PhysicalPlanTranslator.java | 26 ++- .../nereids/parser/LogicalPlanBuilder.java | 78 +++++--- .../doris/nereids/parser/NereidsParser.java | 49 +++-- .../doris/nereids/pattern/PlanPatterns.java | 8 + .../apache/doris/nereids/rules/RuleType.java | 4 +- .../rules/analysis/BindExpression.java | 22 +- .../nereids/rules/analysis/BindSink.java | 4 +- .../rules/expression/ExpressionRewrite.java | 50 +++-- .../expression/rules/ConvertAggStateCast.java | 3 +- .../rules/FoldConstantRuleOnFE.java | 9 + .../nereids/stats/ExpressionEstimation.java | 5 +- .../expressions/literal/DateLiteral.java | 49 ++++- .../expressions/literal/DateTimeLiteral.java | 2 +- .../nereids/trees/plans/Explainable.java | 9 + .../doris/nereids/trees/plans/PlanType.java | 1 + .../trees/plans/algebra/InlineTable.java | 28 +++ .../trees/plans/commands/ExplainCommand.java | 8 +- .../insert/BatchInsertIntoTableCommand.java | 54 ++++- .../insert/FastInsertIntoValuesPlanner.java | 166 +++++++++++++++ .../insert/InsertIntoTableCommand.java | 54 +++-- .../insert/InsertIntoValuesAnalyzer.java | 156 +++++++++++++++ .../insert/InsertOverwriteTableCommand.java | 59 ++++-- .../plans/commands/insert/InsertUtils.java | 189 +++++++++++++++--- .../insert/OlapGroupCommitInsertExecutor.java | 8 +- .../plans/logical/LogicalInlineTable.java | 54 ++++- .../plans/logical/LogicalOneRowRelation.java | 2 +- .../plans/logical/LogicalSetOperation.java | 3 +- .../trees/plans/logical/LogicalUnion.java | 81 ++++++++ .../plans/physical/PhysicalOlapTableSink.java | 2 +- .../trees/plans/visitor/PlanVisitor.java | 6 + .../doris/nereids/util/TypeCoercionUtils.java | 3 +- .../org/apache/doris/qe/AuditLogHelper.java | 6 +- .../org/apache/doris/qe/SessionVariable.java | 21 +- .../SimplifyComparisonPredicateSqlTest.java | 27 ++- .../trees/expressions/SelectReplaceTest.java | 8 +- .../expressions/literal/DateLiteralTest.java | 7 +- .../suites/compression_p0/load.groovy | 2 +- ...test_nestedtypes_insert_into_select.groovy | 4 +- regression-test/suites/index_p0/load.groovy | 2 +- .../insert_group_commit_with_exception.groovy | 2 +- .../dimension/dimension_2_inner_join.groovy | 2 +- 48 files changed, 1217 insertions(+), 228 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPlan.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/InlineTable.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/FastInsertIntoValuesPlanner.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoValuesAnalyzer.java diff --git a/be/src/http/http_channel.cpp b/be/src/http/http_channel.cpp index 312f1ab9286909..598330ff7cbcfb 100644 --- a/be/src/http/http_channel.cpp +++ b/be/src/http/http_channel.cpp @@ -123,7 +123,8 @@ void HttpChannel::send_files(HttpRequest* request, const std::string& root_dir, VLOG_DEBUG << "http channel send file " << file_path << ", size: " << file_size; evbuffer_add_printf(evb.get(), "File-Name: %s\r\n", file.c_str()); - evbuffer_add_printf(evb.get(), "Content-Length: %ld\r\n", file_size); + evbuffer_add_printf(evb.get(), "Content-Length: %" PRIi64 "\r\n", file_size); + evbuffer_add_printf(evb.get(), "\r\n"); if (file_size > 0) { evbuffer_add_file(evb.get(), fd, 0, file_size); diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 368847bac5f270..37e1c68cefb91c 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1476,7 +1476,9 @@ rowConstructor ; rowConstructorItem - : namedExpression | DEFAULT + : constant // duplicate constant rule for improve the parse of `insert into tbl values` + | DEFAULT + | namedExpression ; predicate @@ -1678,7 +1680,7 @@ constant | LEFT_BRACE (items+=constant COLON items+=constant)? (COMMA items+=constant COLON items+=constant)* RIGHT_BRACE #mapLiteral | LEFT_BRACE items+=constant (COMMA items+=constant)* RIGHT_BRACE #structLiteral - | PLACEHOLDER #placeholder + | PLACEHOLDER #placeholder ; comparisonOperator diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 258704763909f1..4f81dde82d97f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -473,9 +473,16 @@ public void setCurrentRootRewriteJobContext(RootRewriteJobContext currentRootRew this.currentRootRewriteJobContext = Optional.ofNullable(currentRootRewriteJobContext); } + /** showPlanProcess */ public boolean showPlanProcess() { Boolean show = showPlanProcess.get(); - return show != null && show; + if (show != null && show) { + return true; + } + if (parent.isPresent()) { + return parent.get().showPlanProcess(); + } + return false; } /** set showPlanProcess in task scope */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 4eafa0e2172f96..6b1c1dd6734435 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -98,17 +98,19 @@ */ public class NereidsPlanner extends Planner { public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class); + + protected Plan parsedPlan; + protected Plan analyzedPlan; + protected Plan rewrittenPlan; + protected Plan optimizedPlan; + protected PhysicalPlan physicalPlan; + private CascadesContext cascadesContext; private final StatementContext statementContext; private final List scanNodeList = Lists.newArrayList(); private final List physicalRelations = Lists.newArrayList(); private DescriptorTable descTable; - private Plan parsedPlan; - private Plan analyzedPlan; - private Plan rewrittenPlan; - private Plan optimizedPlan; - private PhysicalPlan physicalPlan; private FragmentIdMapping distributedPlans; // The cost of optimized plan private double cost = 0; @@ -552,7 +554,7 @@ public Group getRoot() { return cascadesContext.getMemo().getRoot(); } - private PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) { + protected PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) { if (nthPlan <= 1) { cost = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow( () -> new AnalysisException("lowestCostPlans with physicalProperties(" @@ -605,6 +607,9 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical } private long getGarbageCollectionTime() { + if (!ConnectContext.get().getSessionVariable().enableProfile()) { + return 0; + } List gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); long initialGCTime = 0; for (GarbageCollectorMXBean gcBean : gcMxBeans) { @@ -881,7 +886,7 @@ private boolean showPlanProcess(ExplainOptions explainOptions) { return explainOptions != null && explainOptions.showPlanProcess(); } - private void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) { + protected void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) { if (showPlanProcess) { cascadesContext.withPlanProcess(showPlanProcess, task); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java new file mode 100644 index 00000000000000..42d637d676fae2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; +import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** UnboundInlineTable */ +public class UnboundInlineTable extends LogicalLeaf implements InlineTable, BlockFuncDepsPropagation, UnboundPlan { + private final List> constantExprsList; + + public UnboundInlineTable(List> constantExprsList) { + super(PlanType.LOGICAL_UNBOUND_INLINE_TABLE, Optional.empty(), Optional.empty()); + this.constantExprsList = Utils.fastToImmutableList( + Objects.requireNonNull(constantExprsList, "constantExprsList can not be null") + ); + } + + public List> getConstantExprsList() { + return constantExprsList; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitUnboundInlineTable(this, context); + } + + @Override + public List getExpressions() { + ImmutableList.Builder expressions = ImmutableList.builderWithExpectedSize( + constantExprsList.size() * constantExprsList.get(0).size()); + + for (List namedExpressions : constantExprsList) { + expressions.addAll(namedExpressions); + } + + return expressions.build(); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return this; + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return this; + } + + @Override + public List computeOutput() { + throw new UnboundException("output"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPlan.java new file mode 100644 index 00000000000000..2b743f958aaa02 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPlan.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.UnboundLogicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; + +/** UnboundPlan */ +public interface UnboundPlan extends Plan { + @Override + default LogicalProperties computeLogicalProperties() { + return UnboundLogicalProperties.INSTANCE; + } + + @Override + default List computeOutput() { + throw new UnboundException("output"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index 0e528227dc9742..8cf32648d55f05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -34,6 +34,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Objects; @@ -176,4 +177,12 @@ public LogicalProperties computeLogicalProperties() { public List computeOutput() { throw new UnboundException("output"); } + + @Override + public String toString() { + return Utils.toSqlString("UnboundTableSink", + "nameParts", StringUtils.join(nameParts, "."), + "colNames", colNames, + "hints", hints); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index f0fa59977a1902..85243c4b545420 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -207,6 +207,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -2058,17 +2059,22 @@ public PlanFragment visitPhysicalSetOperation( } setOperationNode.setNereidsId(setOperation.getId()); - setOperation.getRegularChildrenOutputs().stream() - .map(o -> o.stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(ImmutableList.toImmutableList())) - .forEach(setOperationNode::addResultExprLists); + for (List regularChildrenOutput : setOperation.getRegularChildrenOutputs()) { + Builder translateOutputs = ImmutableList.builderWithExpectedSize(regularChildrenOutput.size()); + for (SlotReference childOutput : regularChildrenOutput) { + translateOutputs.add(ExpressionTranslator.translate(childOutput, context)); + } + setOperationNode.addResultExprLists(translateOutputs.build()); + } + if (setOperation instanceof PhysicalUnion) { - ((PhysicalUnion) setOperation).getConstantExprsList().stream() - .map(l -> l.stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(ImmutableList.toImmutableList())) - .forEach(setOperationNode::addConstExprList); + for (List unionConsts : ((PhysicalUnion) setOperation).getConstantExprsList()) { + Builder translateConsts = ImmutableList.builderWithExpectedSize(unionConsts.size()); + for (NamedExpression unionConst : unionConsts) { + translateConsts.add(ExpressionTranslator.translate(unionConst, context)); + } + setOperationNode.addConstExprList(translateConsts.build()); + } } for (PlanFragment childFragment : childrenFragments) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 7bc328e238d99d..d98d0660f5c9cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -335,7 +335,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundFunction; -import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -357,6 +357,7 @@ import org.apache.doris.nereids.properties.SelectHintUseMv; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.BitAnd; import org.apache.doris.nereids.trees.expressions.BitNot; @@ -489,6 +490,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; +import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; import org.apache.doris.nereids.trees.plans.commands.AdminCancelRebalanceDiskCommand; @@ -692,10 +695,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalQualify; @@ -887,7 +890,7 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) { } else { throw new ParseException("tableName and tableId cannot both be null"); } - Optional labelName = ctx.labelName == null ? Optional.empty() : Optional.of(ctx.labelName.getText()); + Optional labelName = (ctx.labelName == null) ? Optional.empty() : Optional.of(ctx.labelName.getText()); List colNames = ctx.cols == null ? ImmutableList.of() : visitIdentifierList(ctx.cols); // TODO visit partitionSpecCtx LogicalPlan plan = visitQuery(ctx.query()); @@ -918,7 +921,7 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) { command = new InsertOverwriteTableCommand(sink, labelName, cte); } else { if (ConnectContext.get() != null && ConnectContext.get().isTxnModel() - && sink.child() instanceof LogicalInlineTable + && sink.child() instanceof InlineTable && sink.child().getExpressions().stream().allMatch(Expression::isConstant)) { // FIXME: In legacy, the `insert into select 1` is handled as `insert into values`. // In nereids, the original way is throw an AnalysisException and fallback to legacy. @@ -1848,8 +1851,8 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte LogicalPlan selectPlan; LogicalPlan relation; if (ctx.fromClause() == null) { - relation = new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), - ImmutableList.of(new UnboundAlias(Literal.of(0)))); + relation = new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), + ImmutableList.of(new Alias(Literal.of(0)))); } else { relation = visitFromClause(ctx.fromClause()); } @@ -1879,10 +1882,13 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte @Override public LogicalPlan visitInlineTable(InlineTableContext ctx) { - List> values = ctx.rowConstructor().stream() - .map(this::visitRowConstructor) - .collect(ImmutableList.toImmutableList()); - return new LogicalInlineTable(values); + List rowConstructorContexts = ctx.rowConstructor(); + ImmutableList.Builder> rows + = ImmutableList.builderWithExpectedSize(rowConstructorContexts.size()); + for (RowConstructorContext rowConstructorContext : rowConstructorContexts) { + rows.add(visitRowConstructor(rowConstructorContext)); + } + return new UnboundInlineTable(rows.build()); } /** @@ -2049,18 +2055,22 @@ public Expression visitStar(StarContext ctx) { throw new ParseException("only one replace clause is supported", ctx); } ReplaceContext replaceContext = (ReplaceContext) exceptOrReplace; - List expectAlias = getNamedExpressions(replaceContext.namedExpressionSeq()); - boolean allAlias = expectAlias.stream() - .allMatch(e -> e instanceof UnboundAlias - && ((UnboundAlias) e).getAlias().isPresent()); - if (expectAlias.isEmpty() || !allAlias) { - throw new ParseException( - "only alias is supported in select-replace clause", ctx); + List expectAlias = Lists.newArrayList(); + NamedExpressionSeqContext namedExpressions = replaceContext.namedExpressionSeq(); + for (NamedExpressionContext namedExpressionContext : namedExpressions.namedExpression()) { + if (namedExpressionContext.identifierOrText() == null) { + throw new ParseException("only alias is supported in select-replace clause", ctx); + } + expectAlias.add((NamedExpression) namedExpressionContext.accept(this)); + } + if (expectAlias.isEmpty()) { + throw new ParseException("only alias is supported in select-replace clause", ctx); } finalReplacedAlias = expectAlias; } else { - throw new ParseException("Unsupported except or replace clause: " + exceptOrReplace.getText(), - ctx); + throw new ParseException( + "Unsupported except or replace clause: " + exceptOrReplace.getText(), ctx + ); } } return new UnboundStar(target, finalExpectSlots, finalReplacedAlias); @@ -2081,11 +2091,16 @@ public NamedExpression visitNamedExpression(NamedExpressionContext ctx) { if (ctx.identifierOrText() == null) { if (expression instanceof NamedExpression) { return (NamedExpression) expression; + } else if (expression instanceof Literal) { + return new Alias(expression); } else { return new UnboundAlias(expression); } } String alias = visitIdentifierOrText(ctx.identifierOrText()); + if (expression instanceof Literal) { + return new Alias(expression, alias); + } return new UnboundAlias(expression, alias); }); } @@ -3013,14 +3028,21 @@ public Expression visitParenthesizedExpression(ParenthesizedExpressionContext ct @Override public List visitRowConstructor(RowConstructorContext ctx) { - return ctx.rowConstructorItem().stream() - .map(this::visitRowConstructorItem) - .collect(ImmutableList.toImmutableList()); + List rowConstructorItemContexts = ctx.rowConstructorItem(); + ImmutableList.Builder columns + = ImmutableList.builderWithExpectedSize(rowConstructorItemContexts.size()); + for (RowConstructorItemContext rowConstructorItemContext : rowConstructorItemContexts) { + columns.add(visitRowConstructorItem(rowConstructorItemContext)); + } + return columns.build(); } @Override public NamedExpression visitRowConstructorItem(RowConstructorItemContext ctx) { - if (ctx.DEFAULT() != null) { + ConstantContext constant = ctx.constant(); + if (constant != null) { + return new Alias((Expression) constant.accept(this)); + } else if (ctx.DEFAULT() != null) { return new DefaultValueSlot(); } else { return visitNamedExpression(ctx.namedExpression()); @@ -3565,14 +3587,6 @@ private LogicalPlan withLimit(LogicalPlan input, Optional li }); } - private UnboundOneRowRelation withOneRowRelation(SelectColumnClauseContext selectCtx) { - return ParserUtils.withOrigin(selectCtx, () -> { - // fromClause does not exists. - List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); - return new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), projects); - }); - } - /** * Add a regular (SELECT) query specification to a logical plan. The query specification * is the core of the logical plan, this is where sourcing (FROM clause), projection (SELECT), @@ -3830,7 +3844,7 @@ protected LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContex } } else { List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); - if (input instanceof UnboundOneRowRelation) { + if (input instanceof OneRowRelation) { if (projects.stream().anyMatch(project -> project instanceof UnboundStar)) { throw new ParseException("SELECT * must have a FROM clause"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 4ed71bbbc14673..c273f50b04ac44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -56,6 +56,7 @@ import java.lang.reflect.Method; import java.util.BitSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -326,37 +327,40 @@ private T parse(String sql, Function parseFu private T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder, Function parseFunction) { - ParserRuleContext tree = toAst(sql, parseFunction); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, parseFunction); LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null - ? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder; + ? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint)) + : logicalPlanBuilder; return (T) realLogicalPlanBuilder.visit(tree); } public LogicalPlan parseForCreateView(String sql) { - ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView( - getHintMap(sql, DorisParser::selectHint)); + getHintMap(sql, tokenStream, DorisParser::selectHint)); return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } + /** parseForSyncMv */ public Optional parseForSyncMv(String sql) { - ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv( - getHintMap(sql, DorisParser::selectHint)); + getHintMap(sql, tokenStream, DorisParser::selectHint)); logicalPlanBuilderForSyncMv.visit(tree); return logicalPlanBuilderForSyncMv.getQuerySql(); } /** get hint map */ - public static Map getHintMap(String sql, + public static Map getHintMap(String sql, CommonTokenStream hintTokenStream, Function parseFunction) { // parse hint first round - DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer); - Map selectHintMap = Maps.newHashMap(); - Token hintToken = hintTokenStream.getTokenSource().nextToken(); + Iterator tokenIterator = hintTokenStream.getTokens().iterator(); + Token hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; while (hintToken != null && hintToken.getType() != DorisLexer.EOF) { if (hintToken.getChannel() == 2 && sql.charAt(hintToken.getStartIndex() + 2) == '+') { String hintSql = sql.substring(hintToken.getStartIndex() + 3, hintToken.getStopIndex() + 1); @@ -366,15 +370,19 @@ public static Map getHintMap(String sql, ParserRuleContext hintContext = parseFunction.apply(hintParser); selectHintMap.put(hintToken.getStartIndex(), hintContext); } - hintToken = hintTokenStream.getTokenSource().nextToken(); + hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; } return selectHintMap; } + public static ParserRuleContext toAst( + String sql, Function parseFunction) { + return toAst(parseAllTokens(sql), parseFunction); + } + /** toAst */ - public static ParserRuleContext toAst(String sql, Function parseFunction) { - DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); + public static ParserRuleContext toAst( + CommonTokenStream tokenStream, Function parseFunction) { DorisParser parser = new DorisParser(tokenStream); parser.addParseListener(POST_PROCESSOR); @@ -405,9 +413,7 @@ public static ParserRuleContext toAst(String sql, Function> aggregate() { default PatternDescriptor> aggregate(PatternDescriptor child) { return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor inlineTable() { + return new PatternDescriptor(new TypePattern(InlineTable.class), defaultPromise()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 4cf3c75b68dc43..86d0495b851bd2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -62,6 +62,7 @@ public enum RuleType { BINDING_UNBOUND_TVF_RELATION_FUNCTION(RuleTypeClass.REWRITE), BINDING_SET_OPERATION_SLOT(RuleTypeClass.REWRITE), BINDING_INLINE_TABLE_SLOT(RuleTypeClass.REWRITE), + LOGICAL_INLINE_TABLE_TO_LOGICAL_UNION_OR_ONE_ROW_RELATION(RuleTypeClass.REWRITE), COUNT_LITERAL_REWRITE(RuleTypeClass.REWRITE), SUM_LITERAL_REWRITE(RuleTypeClass.REWRITE), @@ -496,8 +497,7 @@ public enum RuleType { IMPLEMENTATION_SENTINEL(RuleTypeClass.IMPLEMENTATION), // sentinel, use to count rules - SENTINEL(RuleTypeClass.SENTINEL), - ; + SENTINEL(RuleTypeClass.SENTINEL); private final RuleTypeClass ruleTypeClass; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 1e481542baec12..c308a1e7e796e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -66,6 +66,7 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -74,7 +75,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; @@ -195,7 +195,7 @@ protected boolean condition(Rule rule, Plan plan) { logicalQualify(logicalHaving()).thenApply(this::bindQualifyHaving) ), RuleType.BINDING_INLINE_TABLE_SLOT.build( - logicalInlineTable().thenApply(this::bindInlineTable) + inlineTable().thenApply(this::bindInlineTable) ), RuleType.BINDING_ONE_ROW_RELATION_SLOT.build( // we should bind UnboundAlias in the UnboundOneRowRelation @@ -349,24 +349,24 @@ private LogicalOneRowRelation bindOneRowRelation(MatchingContext ctx) { - LogicalInlineTable logicalInlineTable = ctx.root; + private LogicalPlan bindInlineTable(MatchingContext ctx) { + InlineTable inlineTable = ctx.root; // ensure all expressions are valid. + List> constantExprsList = inlineTable.getConstantExprsList(); List relations - = Lists.newArrayListWithCapacity(logicalInlineTable.getConstantExprsList().size()); - for (int i = 0; i < logicalInlineTable.getConstantExprsList().size(); i++) { - for (NamedExpression constantExpr : logicalInlineTable.getConstantExprsList().get(i)) { + = Lists.newArrayListWithCapacity(constantExprsList.size()); + for (int i = 0; i < constantExprsList.size(); i++) { + List row = constantExprsList.get(i); + for (NamedExpression constantExpr : row) { if (constantExpr instanceof DefaultValueSlot) { throw new AnalysisException("Default expression" + " can't exist in SELECT statement at row " + (i + 1)); } } - relations.add(new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), - logicalInlineTable.getConstantExprsList().get(i))); + relations.add(new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), row)); } // construct union all tree - return LogicalPlanBuilder.reduceToLogicalPlanTree(0, relations.size() - 1, - relations, Qualifier.ALL); + return LogicalPlanBuilder.reduceToLogicalPlanTree(0, relations.size() - 1, relations, Qualifier.ALL); } private LogicalHaving bindHaving(MatchingContext> ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 9b4ff631a838b6..f08148d73e7fb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -80,6 +80,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; @@ -253,7 +254,7 @@ private Plan bindOlapTableSink(MatchingContext> ctx) { private LogicalProject getOutputProjectByCoercion(List tableSchema, LogicalPlan child, Map columnToOutput) { - List fullOutputExprs = ImmutableList.copyOf(columnToOutput.values()); + List fullOutputExprs = Utils.fastToImmutableList(columnToOutput.values()); if (child instanceof LogicalOneRowRelation) { // remove default value slot in one row relation child = ((LogicalOneRowRelation) child).withProjects(((LogicalOneRowRelation) child) @@ -274,6 +275,7 @@ private LogicalProject getOutputProjectByCoercion(List tableSchema, L // we skip it. continue; } + expr = expr.toSlot(); DataType inputType = expr.getDataType(); DataType targetType = DataType.fromCatalogType(tableSchema.get(i).getType()); Expression castExpr = expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index e5b74ee26bcb02..0fcc58e0273d2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import java.util.Collection; @@ -85,7 +86,8 @@ public List buildRules() { new OlapTableSinkExpressionRewrite().build()); } - private class GenerateExpressionRewrite extends OneRewriteRuleFactory { + /** GenerateExpressionRewrite */ + public class GenerateExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalGenerate().thenApply(ctx -> { @@ -103,7 +105,8 @@ public Rule build() { } } - private class OneRowRelationExpressionRewrite extends OneRewriteRuleFactory { + /** OneRowRelationExpressionRewrite */ + public class OneRowRelationExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalOneRowRelation().thenApply(ctx -> { @@ -111,19 +114,25 @@ public Rule build() { List projects = oneRowRelation.getProjects(); ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - List newProjects = projects - .stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) - .collect(ImmutableList.toImmutableList()); - if (projects.equals(newProjects)) { - return oneRowRelation; + Builder rewrittenExprs + = ImmutableList.builderWithExpectedSize(projects.size()); + boolean changed = false; + for (NamedExpression project : projects) { + NamedExpression newProject = (NamedExpression) rewriter.rewrite(project, context); + if (!changed && !project.deepEquals(newProject)) { + changed = true; + } + rewrittenExprs.add(newProject); } - return new LogicalOneRowRelation(oneRowRelation.getRelationId(), newProjects); + return changed + ? new LogicalOneRowRelation(oneRowRelation.getRelationId(), rewrittenExprs.build()) + : oneRowRelation; }).toRule(RuleType.REWRITE_ONE_ROW_RELATION_EXPRESSION); } } - private class ProjectExpressionRewrite extends OneRewriteRuleFactory { + /** ProjectExpressionRewrite */ + public class ProjectExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalProject().thenApply(ctx -> { @@ -139,7 +148,8 @@ public Rule build() { } } - private class FilterExpressionRewrite extends OneRewriteRuleFactory { + /** FilterExpressionRewrite */ + public class FilterExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter().thenApply(ctx -> { @@ -155,7 +165,8 @@ public Rule build() { } } - private class OlapTableSinkExpressionRewrite extends OneRewriteRuleFactory { + /** OlapTableSinkExpressionRewrite */ + public class OlapTableSinkExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalOlapTableSink().thenApply(ctx -> { @@ -177,7 +188,8 @@ public Rule build() { } } - private class AggExpressionRewrite extends OneRewriteRuleFactory { + /** AggExpressionRewrite */ + public class AggExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalAggregate().thenApply(ctx -> { @@ -197,7 +209,8 @@ public Rule build() { } } - private class JoinExpressionRewrite extends OneRewriteRuleFactory { + /** JoinExpressionRewrite */ + public class JoinExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalJoin().thenApply(ctx -> { @@ -244,7 +257,8 @@ private Pair> rewriteConjuncts(List conjun } } - private class SortExpressionRewrite extends OneRewriteRuleFactory { + /** SortExpressionRewrite */ + public class SortExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { @@ -265,7 +279,8 @@ public Rule build() { } } - private class HavingExpressionRewrite extends OneRewriteRuleFactory { + /** HavingExpressionRewrite */ + public class HavingExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalHaving().thenApply(ctx -> { @@ -281,7 +296,8 @@ public Rule build() { } } - private class LogicalRepeatRewrite extends OneRewriteRuleFactory { + /** LogicalRepeatRewrite */ + public class LogicalRepeatRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalRepeat().thenApply(ctx -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java index 6aa4529ddd4ab6..6d5a70139ab19c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java @@ -47,7 +47,8 @@ public List> buildRules() { ); } - private static Expression convert(Cast cast) { + /** convert */ + public static Expression convert(Cast cast) { Expression child = cast.child(); DataType originalType = child.getDataType(); DataType targetType = cast.getDataType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index d1c385ec621062..c439458ff4c96e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -24,7 +24,9 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.analyzer.UnboundVariable; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule; import org.apache.doris.nereids.rules.expression.ExpressionListenerMatcher; import org.apache.doris.nereids.rules.expression.ExpressionMatchingContext; @@ -53,6 +55,7 @@ import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; +import org.apache.doris.nereids.trees.expressions.Variable; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; @@ -220,6 +223,12 @@ public Expression visitMatch(Match match, ExpressionRewriteContext context) { return super.visitMatch(match, context); } + @Override + public Expression visitUnboundVariable(UnboundVariable unboundVariable, ExpressionRewriteContext context) { + Variable variable = ExpressionAnalyzer.resolveUnboundVariable(unboundVariable); + return variable.getRealExpression(); + } + @Override public Expression visitEncryptKeyRef(EncryptKeyRef encryptKeyRef, ExpressionRewriteContext context) { String dbName = encryptKeyRef.getDbName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java index 7d1b5439bace23..825bb6f7180d16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java @@ -101,6 +101,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.collections.CollectionUtils; +import java.time.DateTimeException; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -212,7 +213,7 @@ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType long min = dateMinLiteral.getValue(); builder.setMinValue(min); builder.setMinExpr(dateMinLiteral.toLegacyLiteral()); - } catch (AnalysisException e) { + } catch (AnalysisException | DateTimeException e) { convertSuccess = false; } } @@ -223,7 +224,7 @@ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType long max = dateMaxLiteral.getValue(); builder.setMaxValue(max); builder.setMaxExpr(dateMaxLiteral.toLegacyLiteral()); - } catch (AnalysisException e) { + } catch (AnalysisException | DateTimeException e) { convertSuccess = false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index ed99e3025e8603..eb8269d68fd0a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableSet; +import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Year; @@ -269,8 +270,8 @@ static Result normalize(String s) { } /** parseDateLiteral */ - public static Result parseDateLiteral(String s) { - Result parseResult = parseDateTime(s); + public static Result parseDateLiteral(String s) { + Result parseResult = parseDateTime(s); if (parseResult.isError()) { return parseResult.cast(); } @@ -286,17 +287,24 @@ public static Result parseDateLiteral(String s) } /** parseDateTime */ - public static Result parseDateTime(String s) { - // fast parse '2022-01-01' - if (s.length() == 10 && s.charAt(4) == '-' && s.charAt(7) == '-') { - TemporalAccessor date = fastParseDate(s); - if (date != null) { - return Result.ok(date); - } - } - + public static Result parseDateTime(String s) { String originalString = s; try { + // fast parse '2022-01-01' + if ((s.length() == 10 || s.length() == 19) && s.charAt(4) == '-' && s.charAt(7) == '-') { + if (s.length() == 10) { + TemporalAccessor date = fastParseDate(s); + if (date != null) { + return Result.ok(date); + } + } else if (s.charAt(10) == ' ' && s.charAt(13) == ':' && s.charAt(16) == ':') { + TemporalAccessor date = fastParseDateTime(s); + if (date != null) { + return Result.ok(date); + } + } + } + TemporalAccessor dateTime; // remove suffix/prefix ' ' @@ -342,6 +350,10 @@ public static Result parseDateTime(String s } return Result.ok(dateTime); + } catch (DateTimeException e) { + return Result.err(() -> + new DateTimeException("date/datetime literal [" + originalString + "] is invalid", e) + ); } catch (Exception ex) { return Result.err(() -> new AnalysisException("date/datetime literal [" + originalString + "] is invalid")); } @@ -566,6 +578,21 @@ private static TemporalAccessor fastParseDate(String date) { } } + private static TemporalAccessor fastParseDateTime(String date) { + Integer year = readNextInt(date, 0, 4); + Integer month = readNextInt(date, 5, 2); + Integer day = readNextInt(date, 8, 2); + Integer hour = readNextInt(date, 11, 2); + Integer minute = readNextInt(date, 14, 2); + Integer second = readNextInt(date, 17, 2); + + if (year != null && month != null && day != null && hour != null && minute != null && second != null) { + return LocalDateTime.of(year, month, day, hour, minute, second); + } else { + return null; + } + } + private static Integer readNextInt(String str, int offset, int readLength) { int value = 0; int realReadLength = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 169ed421934824..7912142f97feb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -132,7 +132,7 @@ public static int determineScale(String s) { /** parseDateTimeLiteral */ public static Result parseDateTimeLiteral(String s, boolean isV2) { - Result parseResult = parseDateTime(s); + Result parseResult = parseDateTime(s); if (parseResult.isError()) { return parseResult.cast(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java index 46771392e59cd9..77eef860b98a77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java @@ -17,11 +17,20 @@ package org.apache.doris.nereids.trees.plans; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; +import java.util.Optional; + /** * plan can be explained. */ public interface Explainable { Plan getExplainPlan(ConnectContext ctx) throws Exception; + + default Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) throws Exception { + return Optional.empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 407610fbe08add..2860ec10092312 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -43,6 +43,7 @@ public enum PlanType { LOGICAL_UNBOUND_ONE_ROW_RELATION, LOGICAL_UNBOUND_RELATION, LOGICAL_UNBOUND_TVF_RELATION, + LOGICAL_UNBOUND_INLINE_TABLE, // logical sinks LOGICAL_FILE_SINK, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/InlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/InlineTable.java new file mode 100644 index 00000000000000..0aded14ca77119 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/InlineTable.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.algebra; + +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.LeafPlan; + +import java.util.List; + +/** InlineTable */ +public interface InlineTable extends LeafPlan { + List> getConstantExprsList(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index e3f2f1d732ae5a..ea805f6cb0ceb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -79,12 +79,16 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (!(logicalPlan instanceof Explainable)) { throw new AnalysisException(logicalPlan.getClass().getSimpleName() + " cannot be explained"); } - explainPlan = ((LogicalPlan) ((Explainable) logicalPlan).getExplainPlan(ctx)); + Explainable explainable = (Explainable) logicalPlan; + explainPlan = ((LogicalPlan) explainable.getExplainPlan(ctx)); + NereidsPlanner planner = explainable.getExplainPlanner(explainPlan, ctx.getStatementContext()).orElseGet(() -> + new NereidsPlanner(ctx.getStatementContext()) + ); + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(explainPlan, ctx.getStatementContext()); ExplainOptions explainOptions = new ExplainOptions(level, showPlanProcess); logicalPlanAdapter.setIsExplain(explainOptions); executor.setParsedStmt(logicalPlanAdapter); - NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) { ctx.getStatementContext().addPlannerHook(InitMaterializationContextHook.INSTANCE); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index b4a7a9eee3a148..4fb42a21fd780d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -26,19 +26,22 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.NoForward; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; @@ -69,16 +72,34 @@ public class BatchInsertIntoTableCommand extends Command implements NoForward, E public static final Logger LOG = LogManager.getLogger(BatchInsertIntoTableCommand.class); - private LogicalPlan logicalQuery; + private LogicalPlan originLogicalQuery; + private Optional logicalQuery; public BatchInsertIntoTableCommand(LogicalPlan logicalQuery) { super(PlanType.BATCH_INSERT_INTO_TABLE_COMMAND); - this.logicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.originLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.logicalQuery = Optional.empty(); + } + + public LogicalPlan getLogicalQuery() { + return logicalQuery.orElse(originLogicalQuery); } @Override public Plan getExplainPlan(ConnectContext ctx) throws Exception { - return InsertUtils.getPlanForExplain(ctx, this.logicalQuery); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + return InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery()); + } + + @Override + public Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) throws Exception { + ConnectContext connectContext = ctx.getConnectContext(); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, connectContext); + boolean supportFastInsertIntoValues + = InsertUtils.supportFastInsertIntoValues(logicalPlan, targetTableIf, connectContext); + return Optional.of(new FastInsertIntoValuesPlanner(ctx, supportFastInsertIntoValues)); } @Override @@ -88,19 +109,32 @@ public R accept(PlanVisitor visitor, C context) { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - UnboundTableSink unboundTableSink = (UnboundTableSink) logicalQuery; + UnboundTableSink unboundTableSink = (UnboundTableSink) originLogicalQuery; Plan query = unboundTableSink.child(); - if (!(query instanceof LogicalInlineTable)) { + if (!(query instanceof InlineTable)) { throw new AnalysisException("Insert into ** select is not supported in a transaction"); } PhysicalOlapTableSink sink; - TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, ctx); targetTableIf.readLock(); try { - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); - LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); - NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + StatementContext statementContext = ctx.getStatementContext(); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(statementContext, originLogicalQuery, PhysicalProperties.ANY) + ); + + this.logicalQuery = Optional.of((LogicalPlan) InsertUtils.normalizePlan( + originLogicalQuery, targetTableIf, analyzeContext, Optional.empty() + )); + + LogicalPlan logicalQuery = this.logicalQuery.get(); + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, statementContext); + + boolean supportFastInsertIntoValues + = InsertUtils.supportFastInsertIntoValues(logicalQuery, targetTableIf, ctx); + FastInsertIntoValuesPlanner planner = new FastInsertIntoValuesPlanner( + statementContext, supportFastInsertIntoValues, true); planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); executor.checkBlockRules(); if (ctx.getConnectType() == ConnectType.MYSQL && ctx.getMysqlChannel() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/FastInsertIntoValuesPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/FastInsertIntoValuesPlanner.java new file mode 100644 index 00000000000000..18dcbf25d28aa6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/FastInsertIntoValuesPlanner.java @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.GroupId; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; + +import java.util.concurrent.atomic.AtomicReference; + +/** FastInsertIntoValuesPlanner */ +public class FastInsertIntoValuesPlanner extends NereidsPlanner { + private static final Rule toPhysicalOlapTableSink = new LogicalOlapTableSinkToPhysicalOlapTableSink() + .build(); + protected final boolean fastInsertIntoValues; + protected final boolean batchInsert; + private final AtomicReference rootGroupRef = new AtomicReference<>(); + + public FastInsertIntoValuesPlanner(StatementContext statementContext, boolean fastInsertIntoValues) { + this(statementContext, fastInsertIntoValues, false); + } + + public FastInsertIntoValuesPlanner( + StatementContext statementContext, boolean fastInsertIntoValues, boolean batchInsert) { + super(statementContext); + this.fastInsertIntoValues = fastInsertIntoValues; + this.batchInsert = batchInsert; + } + + @Override + protected void analyze(boolean showPlanProcess) { + if (!fastInsertIntoValues) { + super.analyze(showPlanProcess); + return; + } + CascadesContext cascadesContext = getCascadesContext(); + keepOrShowPlanProcess(showPlanProcess, () -> { + InsertIntoValuesAnalyzer analyzer = new InsertIntoValuesAnalyzer(cascadesContext, batchInsert); + analyzer.execute(); + }); + } + + @Override + protected void rewrite(boolean showPlanProcess) { + if (!fastInsertIntoValues) { + super.rewrite(showPlanProcess); + } + } + + @Override + protected void optimize() { + if (!fastInsertIntoValues) { + super.optimize(); + return; + } + + DefaultPlanRewriter optimizer = new DefaultPlanRewriter() { + @Override + public Plan visitLogicalUnion(LogicalUnion logicalUnion, Void context) { + logicalUnion = (LogicalUnion) super.visitLogicalUnion(logicalUnion, context); + + return new PhysicalUnion(logicalUnion.getQualifier(), + logicalUnion.getOutputs(), + logicalUnion.getRegularChildrenOutputs(), + logicalUnion.getConstantExprsList(), + logicalUnion.getLogicalProperties(), + logicalUnion.children() + ); + } + + @Override + public Plan visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, Void context) { + return new PhysicalOneRowRelation( + oneRowRelation.getRelationId(), + oneRowRelation.getProjects(), + oneRowRelation.getLogicalProperties()); + } + + @Override + public Plan visitLogicalProject(LogicalProject logicalProject, Void context) { + logicalProject = + (LogicalProject) super.visitLogicalProject(logicalProject, context); + + return new PhysicalProject<>( + logicalProject.getProjects(), + logicalProject.getLogicalProperties(), + logicalProject.child() + ); + } + + @Override + public Plan visitLogicalOlapTableSink(LogicalOlapTableSink olapTableSink, + Void context) { + olapTableSink = + (LogicalOlapTableSink) super.visitLogicalOlapTableSink(olapTableSink, context); + return toPhysicalOlapTableSink + .transform(olapTableSink, getCascadesContext()) + .get(0); + } + }; + + PhysicalPlan physicalPlan = + (PhysicalPlan) getCascadesContext().getRewritePlan().accept(optimizer, null); + + super.physicalPlan = physicalPlan; + + GroupId rootGroupId = GroupId.createGenerator().getNextId(); + Group rootGroup = new Group(rootGroupId, physicalPlan.getLogicalProperties()); + rootGroupRef.set(rootGroup); + } + + @Override + public Group getRoot() { + if (!fastInsertIntoValues) { + return super.getRoot(); + } + return rootGroupRef.get(); + } + + @Override + protected PhysicalPlan chooseNthPlan( + Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) { + if (!fastInsertIntoValues) { + return super.chooseNthPlan(rootGroup, physicalProperties, nthPlan); + } + return super.physicalPlan; + } + + @Override + protected PhysicalPlan postProcess(PhysicalPlan physicalPlan) { + if (!fastInsertIntoValues) { + return super.postProcess(physicalPlan); + } + return physicalPlan; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 10f9947974cdb0..96d5d56a7e10ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -31,11 +31,13 @@ import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.load.loadv2.LoadStatistic; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; @@ -85,8 +87,8 @@ public class InsertIntoTableCommand extends Command implements ForwardWithSync, public static final Logger LOG = LogManager.getLogger(InsertIntoTableCommand.class); - private LogicalPlan originalLogicalQuery; - private LogicalPlan logicalQuery; + private LogicalPlan originLogicalQuery; + private Optional logicalQuery; private Optional labelName; /** * When source it's from job scheduler,it will be set. @@ -101,15 +103,15 @@ public class InsertIntoTableCommand extends Command implements ForwardWithSync, public InsertIntoTableCommand(LogicalPlan logicalQuery, Optional labelName, Optional insertCtx, Optional cte) { super(PlanType.INSERT_INTO_TABLE_COMMAND); - this.originalLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); - this.logicalQuery = originalLogicalQuery; + this.originLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); this.labelName = Objects.requireNonNull(labelName, "labelName should not be null"); + this.logicalQuery = Optional.empty(); this.insertCtx = insertCtx; this.cte = cte; } public LogicalPlan getLogicalQuery() { - return logicalQuery; + return logicalQuery.orElse(originLogicalQuery); } public Optional getLabelName() { @@ -149,7 +151,7 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor */ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor stmtExecutor, boolean needBeginTransaction) throws Exception { - List qualifiedTargetTableName = InsertUtils.getTargetTableQualified(logicalQuery, ctx); + List qualifiedTargetTableName = InsertUtils.getTargetTableQualified(originLogicalQuery, ctx); AbstractInsertExecutor insertExecutor; int retryTimes = 0; @@ -214,8 +216,6 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor stmtExec // so we need to set this here insertExecutor.getCoordinator().setTxnId(insertExecutor.getTxnId()); stmtExecutor.setCoord(insertExecutor.getCoordinator()); - // for prepare and execute, avoiding normalization for every execute command - this.originalLogicalQuery = this.logicalQuery; return insertExecutor; } LOG.warn("insert plan failed {} times. query id is {}.", retryTimes, DebugUtil.printId(ctx.queryId())); @@ -226,17 +226,23 @@ private BuildInsertExecutorResult initPlanOnce(ConnectContext ctx, StmtExecutor stmtExecutor, TableIf targetTableIf) throws Throwable { targetTableIf.readLock(); try { + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); // process inline table (default values, empty values) - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(originalLogicalQuery, targetTableIf, insertCtx); + this.logicalQuery = Optional.of((LogicalPlan) InsertUtils.normalizePlan( + originLogicalQuery, targetTableIf, analyzeContext, insertCtx + )); if (cte.isPresent()) { - this.logicalQuery = ((LogicalPlan) cte.get().withChildren(logicalQuery)); + this.logicalQuery = Optional.of((LogicalPlan) cte.get().withChildren(logicalQuery.get())); } - OlapGroupCommitInsertExecutor.analyzeGroupCommit(ctx, targetTableIf, this.logicalQuery, this.insertCtx); + OlapGroupCommitInsertExecutor.analyzeGroupCommit( + ctx, targetTableIf, this.logicalQuery.get(), this.insertCtx); } finally { targetTableIf.readUnlock(); } - LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery.get(), ctx.getStatementContext()); return planInsertExecutor(ctx, stmtExecutor, logicalPlanAdapter, targetTableIf); } @@ -362,6 +368,9 @@ private ExecutorFactory selectInsertExecutorFactory( private BuildInsertExecutorResult planInsertExecutor( ConnectContext ctx, StmtExecutor stmtExecutor, LogicalPlanAdapter logicalPlanAdapter, TableIf targetTableIf) throws Throwable { + LogicalPlan logicalPlan = logicalPlanAdapter.getLogicalPlan(); + + boolean supportFastInsertIntoValues = InsertUtils.supportFastInsertIntoValues(logicalPlan, targetTableIf, ctx); // the key logical when use new coordinator: // 1. use NereidsPlanner to generate PhysicalPlan // 2. use PhysicalPlan to select InsertExecutorFactory, some InsertExecutors want to control @@ -372,10 +381,9 @@ private BuildInsertExecutorResult planInsertExecutor( // 3. NereidsPlanner use PhysicalPlan and the provided backend to generate DistributePlan // 4. ExecutorFactory use the DistributePlan to generate the NereidsSqlCoordinator and InsertExecutor - StatementContext statementContext = ctx.getStatementContext(); - AtomicReference executorFactoryRef = new AtomicReference<>(); - NereidsPlanner planner = new NereidsPlanner(statementContext) { + FastInsertIntoValuesPlanner planner = new FastInsertIntoValuesPlanner( + ctx.getStatementContext(), supportFastInsertIntoValues) { @Override protected void doDistribute(boolean canUseNereidsDistributePlanner) { // when enter this method, the step 1 already executed @@ -406,12 +414,24 @@ private void runInternal(ConnectContext ctx, StmtExecutor executor) throws Excep } public boolean isExternalTableSink() { - return !(logicalQuery instanceof UnboundTableSink); + return !(getLogicalQuery() instanceof UnboundTableSink); } @Override public Plan getExplainPlan(ConnectContext ctx) { - return InsertUtils.getPlanForExplain(ctx, this.logicalQuery); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + return InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery()); + } + + @Override + public Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) { + ConnectContext connectContext = ctx.getConnectContext(); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, connectContext); + boolean supportFastInsertIntoValues + = InsertUtils.supportFastInsertIntoValues(logicalPlan, targetTableIf, connectContext); + return Optional.of(new FastInsertIntoValuesPlanner(ctx, supportFastInsertIntoValues)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoValuesAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoValuesAnalyzer.java new file mode 100644 index 00000000000000..1c630a41c846fe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoValuesAnalyzer.java @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor; +import org.apache.doris.nereids.jobs.rewrite.RewriteJob; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.BindSink; +import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteRule; +import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; +import org.apache.doris.nereids.rules.rewrite.MergeProjects; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation; +import org.apache.doris.nereids.rules.rewrite.PushProjectIntoUnion; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.types.DataType; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; + +/** InsertIntoValuesAnalyzer */ +public class InsertIntoValuesAnalyzer extends AbstractBatchJobExecutor { + public static final List INSERT_JOBS = jobs( + bottomUp( + new InlineTableToUnionOrOneRowRelation(), + new BindSink(), + new MergeProjects(), + // after bind olap table sink, the LogicalProject will be generated under LogicalOlapTableSink, + // we should convert the agg state function in the project, and evaluate some env parameters + // like encrypt key reference, for example: `values (aes_encrypt("abc",key test.my_key))`, + // we should replace the `test.my_key` to real key + new RewriteInsertIntoExpressions(ExpressionRewrite.bottomUp( + ConvertAggStateCast.INSTANCE, + FoldConstantRuleOnFE.PATTERN_MATCH_INSTANCE + )) + ) + ); + + public static final List BATCH_INSERT_JOBS = jobs( + bottomUp( + new InlineTableToUnionOrOneRowRelation(), + new BindSink(), + new MergeProjects(), + + // the BatchInsertIntoTableCommand need send StringLiteral to backend, + // and only support alias(literal as xx) or alias(cast(literal as xx)), + // but not support alias(cast(slotRef as xx)) which create in BindSink, + // we should push down the cast into Union or OneRowRelation. + // the InsertIntoTableCommand support translate slotRef in the TPlan, + // so we don't need this rules, just evaluate in backend + new PushProjectIntoUnion(), + new PushProjectIntoOneRowRelation(), + + new RewriteBatchInsertIntoExpressions(ExpressionRewrite.bottomUp( + ConvertAggStateCast.INSTANCE, + FoldConstantRuleOnFE.PATTERN_MATCH_INSTANCE + )) + ) + ); + + private final boolean batchInsert; + + public InsertIntoValuesAnalyzer(CascadesContext cascadesContext, boolean batchInsert) { + super(cascadesContext); + this.batchInsert = batchInsert; + } + + @Override + public List getJobs() { + return batchInsert ? BATCH_INSERT_JOBS : INSERT_JOBS; + } + + // we only rewrite the project's expression + private static class RewriteInsertIntoExpressions extends ExpressionRewrite { + public RewriteInsertIntoExpressions(ExpressionRewriteRule... rules) { + super(rules); + } + + @Override + public List buildRules() { + return ImmutableList.of( + new ProjectExpressionRewrite().build() + ); + } + } + + // we only rewrite the project's and one row relation expression + private static class RewriteBatchInsertIntoExpressions extends ExpressionRewrite { + public RewriteBatchInsertIntoExpressions(ExpressionRewriteRule... rules) { + super(rules); + } + + @Override + public List buildRules() { + return ImmutableList.of( + new ProjectExpressionRewrite().build(), + new OneRowRelationExpressionRewrite().build() + ); + } + } + + private static class InlineTableToUnionOrOneRowRelation extends OneRewriteRuleFactory { + @Override + public Rule build() { + return inlineTable().then(inlineTable -> { + List> originConstants = inlineTable.getConstantExprsList(); + if (originConstants.size() > 1) { + Pair>, List> castedConstantsAndNullables + = LogicalUnion.castCommonDataTypeAndNullableByConstants(originConstants); + List> castedRows = castedConstantsAndNullables.key(); + List nullables = castedConstantsAndNullables.value(); + List outputs = Lists.newArrayList(); + List firstRow = originConstants.get(0); + for (int columnId = 0; columnId < firstRow.size(); columnId++) { + String name = firstRow.get(columnId).getName(); + DataType commonDataType = castedRows.get(0).get(columnId).getDataType(); + outputs.add(new SlotReference(name, commonDataType, nullables.get(columnId))); + } + return new LogicalUnion(Qualifier.ALL, castedRows, ImmutableList.of()).withNewOutputs(outputs); + } else if (originConstants.size() == 1) { + return new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), originConstants.get(0)); + } else { + throw new AnalysisException("Illegal inline table with empty constants"); + } + }).toRule(RuleType.LOGICAL_INLINE_TABLE_TO_LOGICAL_UNION_OR_ONE_ROW_RELATION); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index c89a4fc7be96ee..68c71de2d9e8b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -32,13 +32,16 @@ import org.apache.doris.insertoverwrite.InsertOverwriteUtil; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; @@ -82,7 +85,8 @@ public class InsertOverwriteTableCommand extends Command implements ForwardWithS private static final Logger LOG = LogManager.getLogger(InsertOverwriteTableCommand.class); - private LogicalPlan logicalQuery; + private LogicalPlan originLogicalQuery; + private Optional logicalQuery; private Optional labelName; private final Optional cte; private AtomicBoolean isCancelled = new AtomicBoolean(false); @@ -94,7 +98,8 @@ public class InsertOverwriteTableCommand extends Command implements ForwardWithS public InsertOverwriteTableCommand(LogicalPlan logicalQuery, Optional labelName, Optional cte) { super(PlanType.INSERT_INTO_TABLE_COMMAND); - this.logicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.originLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.logicalQuery = Optional.empty(); this.labelName = Objects.requireNonNull(labelName, "labelName should not be null"); this.cte = cte; } @@ -103,14 +108,18 @@ public void setLabelName(Optional labelName) { this.labelName = labelName; } - public boolean isAutoDetectOverwrite() { + public boolean isAutoDetectOverwrite(LogicalPlan logicalQuery) { return (logicalQuery instanceof UnboundTableSink) - && ((UnboundTableSink) this.logicalQuery).isAutoDetectPartition(); + && ((UnboundTableSink) logicalQuery).isAutoDetectPartition(); + } + + public LogicalPlan getLogicalQuery() { + return logicalQuery.orElse(originLogicalQuery); } @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, ctx); //check allow insert overwrite if (!allowInsertOverwrite(targetTableIf)) { String errMsg = "insert into overwrite only support OLAP and HMS/ICEBERG table." @@ -122,12 +131,20 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (targetTableIf instanceof MTMV && !MTMVUtil.allowModifyMTMVData(ctx)) { throw new AnalysisException("Not allowed to perform current operation on async materialized view"); } - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + this.logicalQuery = Optional.of((LogicalPlan) InsertUtils.normalizePlan( + originLogicalQuery, targetTableIf, analyzeContext, Optional.empty())); if (cte.isPresent()) { - this.logicalQuery = (LogicalPlan) logicalQuery.withChildren(cte.get().withChildren( - this.logicalQuery.child(0))); + LogicalPlan logicalQuery = this.logicalQuery.get(); + this.logicalQuery = Optional.of( + (LogicalPlan) logicalQuery.withChildren( + cte.get().withChildren(logicalQuery.child(0)) + ) + ); } - + LogicalPlan logicalQuery = this.logicalQuery.get(); LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); @@ -172,7 +189,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { isRunning.set(true); long taskId = 0; try { - if (isAutoDetectOverwrite()) { + if (isAutoDetectOverwrite(getLogicalQuery())) { // taskId here is a group id. it contains all replace tasks made and registered in rpc process. taskId = insertOverwriteManager.registerTaskGroup(); // When inserting, BE will call to replace partition by FrontendService. FE will register new temp @@ -219,7 +236,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { } } catch (Exception e) { LOG.warn("insert into overwrite failed with task(or group) id " + taskId); - if (isAutoDetectOverwrite()) { + if (isAutoDetectOverwrite(getLogicalQuery())) { insertOverwriteManager.taskGroupFail(taskId); } else { insertOverwriteManager.taskFail(taskId); @@ -287,6 +304,7 @@ private void insertIntoPartitions(ConnectContext ctx, StmtExecutor executor, Lis // copy sink tot replace by tempPartitions UnboundLogicalSink copySink; InsertCommandContext insertCtx; + LogicalPlan logicalQuery = getLogicalQuery(); if (logicalQuery instanceof UnboundTableSink) { UnboundTableSink sink = (UnboundTableSink) logicalQuery; copySink = (UnboundLogicalSink) UnboundTableSinkCreator.createUnboundTableSink( @@ -342,6 +360,7 @@ private void insertIntoPartitions(ConnectContext ctx, StmtExecutor executor, Lis */ private void insertIntoAutoDetect(ConnectContext ctx, StmtExecutor executor, long groupId) throws Exception { InsertCommandContext insertCtx; + LogicalPlan logicalQuery = getLogicalQuery(); if (logicalQuery instanceof UnboundTableSink) { // 1. when overwrite auto-detect, allow auto partition or not is controlled by session variable. // 2. we save and pass overwrite auto detect by insertCtx @@ -362,7 +381,23 @@ private void insertIntoAutoDetect(ConnectContext ctx, StmtExecutor executor, lon @Override public Plan getExplainPlan(ConnectContext ctx) { - return InsertUtils.getPlanForExplain(ctx, this.logicalQuery); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + return InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery()); + } + + @Override + public Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) { + LogicalPlan logicalQuery = getLogicalQuery(); + if (logicalQuery instanceof UnboundTableSink) { + boolean allowAutoPartition = ctx.getConnectContext().getSessionVariable().isEnableAutoCreateWhenOverwrite(); + OlapInsertCommandContext insertCtx = new OlapInsertCommandContext(allowAutoPartition, true); + InsertIntoTableCommand insertIntoTableCommand = new InsertIntoTableCommand( + logicalQuery, labelName, Optional.of(insertCtx), Optional.empty()); + return insertIntoTableCommand.getExplainPlanner(logicalPlan, ctx); + } + return Optional.empty(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 459ffcd04f894a..497a287e802823 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -29,30 +29,40 @@ import org.apache.doris.common.FormatOptions; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalTable; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundAlias; +import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; import org.apache.doris.nereids.analyzer.UnboundJdbcTableSink; -import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.analyzer.UnboundTableSink; +import org.apache.doris.nereids.analyzer.UnboundVariable; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.parser.LogicalPlanBuilder; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; +import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.TypeCoercionUtils; @@ -79,7 +89,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; @@ -260,7 +272,9 @@ private static void beginBatchInsertTransaction(ConnectContext ctx, /** * normalize plan to let it could be process correctly by nereids */ - public static Plan normalizePlan(Plan plan, TableIf table, Optional insertCtx) { + public static Plan normalizePlan(LogicalPlan plan, TableIf table, + Optional analyzeContext, + Optional insertCtx) { UnboundLogicalSink unboundLogicalSink = (UnboundLogicalSink) plan; if (table instanceof HMSExternalTable) { HMSExternalTable hiveTable = (HMSExternalTable) table; @@ -334,21 +348,39 @@ public static Plan normalizePlan(Plan plan, TableIf table, Optional oneRowRelationBuilder = ImmutableList.builder(); + + UnboundInlineTable unboundInlineTable = (UnboundInlineTable) query; + ImmutableList.Builder> optimizedRowConstructors + = ImmutableList.builderWithExpectedSize(unboundInlineTable.getConstantExprsList().size()); List columns = table.getBaseSchema(false); - for (List values : logicalInlineTable.getConstantExprsList()) { - ImmutableList.Builder constantExprs = ImmutableList.builder(); + ConnectContext context = ConnectContext.get(); + ExpressionRewriteContext rewriteContext = null; + if (context != null && context.getStatementContext() != null) { + rewriteContext = new ExpressionRewriteContext( + CascadesContext.initContext( + context.getStatementContext(), unboundInlineTable, PhysicalProperties.ANY + ) + ); + } + + Optional analyzer = analyzeContext.map( + cascadesContext -> buildExprAnalyzer(plan, cascadesContext) + ); + + for (List values : unboundInlineTable.getConstantExprsList()) { + ImmutableList.Builder optimizedRowConstructor = ImmutableList.builder(); if (values.isEmpty()) { if (CollectionUtils.isNotEmpty(unboundLogicalSink.getColNames())) { throw new AnalysisException("value list should not be empty if columns are specified"); } - for (Column column : columns) { - constantExprs.add(generateDefaultExpression(column)); + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); + NamedExpression defaultExpression = generateDefaultExpression(column); + addColumnValue(analyzer, optimizedRowConstructor, defaultExpression); } } else { if (CollectionUtils.isNotEmpty(unboundLogicalSink.getColNames())) { @@ -374,10 +406,15 @@ public static Plan normalizePlan(Plan plan, TableIf table, Optional oneRowRelations = oneRowRelationBuilder.build(); - if (oneRowRelations.size() == 1) { - return plan.withChildren(oneRowRelations.get(0)); - } else { - return plan.withChildren( - LogicalPlanBuilder.reduceToLogicalPlanTree(0, oneRowRelations.size() - 1, - oneRowRelations, Qualifier.ALL)); + return plan.withChildren(new LogicalInlineTable(optimizedRowConstructors.build())); + } + + /** buildAnalyzer */ + public static ExpressionAnalyzer buildExprAnalyzer(Plan plan, CascadesContext analyzeContext) { + return new ExpressionAnalyzer(plan, new Scope(ImmutableList.of()), + analyzeContext, false, false) { + @Override + public Expression visitCast(Cast cast, ExpressionRewriteContext context) { + Expression expr = super.visitCast(cast, context); + if (expr instanceof Cast) { + if (expr.child(0).getDataType() instanceof AggStateType) { + expr = ConvertAggStateCast.convert((Cast) expr); + } else { + expr = FoldConstantRuleOnFE.evaluate(expr, context); + } + } + return expr; + } + + @Override + public Expression visitUnboundFunction(UnboundFunction unboundFunction, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundFunction(unboundFunction, context); + if (expr instanceof UnboundFunction) { + throw new IllegalStateException("Can not analyze function " + unboundFunction.getName()); + } + return expr; + } + + @Override + public Expression visitUnboundSlot(UnboundSlot unboundSlot, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundSlot(unboundSlot, context); + if (expr instanceof UnboundFunction) { + throw new AnalysisException("Can not analyze slot " + unboundSlot.getName()); + } + return expr; + } + + @Override + public Expression visitUnboundVariable(UnboundVariable unboundVariable, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundVariable(unboundVariable, context); + if (expr instanceof UnboundVariable) { + throw new AnalysisException("Can not analyze variable " + unboundVariable.getName()); + } + return expr; + } + + @Override + public Expression visitUnboundAlias(UnboundAlias unboundAlias, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundAlias(unboundAlias, context); + if (expr instanceof UnboundVariable) { + throw new AnalysisException("Can not analyze alias"); + } + return expr; + } + + @Override + public Expression visitUnboundStar(UnboundStar unboundStar, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundStar(unboundStar, context); + if (expr instanceof UnboundStar) { + List qualifier = unboundStar.getQualifier(); + List qualified = new ArrayList<>(qualifier); + qualified.add("*"); + throw new AnalysisException("Can not analyze " + StringUtils.join(qualified, ".")); + } + return expr; + } + }; + } + + private static void addColumnValue( + Optional analyzer, + ImmutableList.Builder optimizedRowConstructor, + NamedExpression value) { + if (analyzer.isPresent() && !(value instanceof Alias && value.child(0) instanceof Literal)) { + ExpressionAnalyzer expressionAnalyzer = analyzer.get(); + value = (NamedExpression) expressionAnalyzer.analyze( + value, new ExpressionRewriteContext(expressionAnalyzer.getCascadesContext()) + ); } + optimizedRowConstructor.add(value); } private static Expression castValue(Expression value, DataType targetType) { - if (value instanceof UnboundAlias) { - return value.withChildren(TypeCoercionUtils.castUnbound(((UnboundAlias) value).child(), targetType)); + if (value instanceof Alias) { + Expression oldChild = value.child(0); + Expression newChild = TypeCoercionUtils.castUnbound(oldChild, targetType); + return oldChild == newChild ? value : value.withChildren(newChild); + } else if (value instanceof UnboundAlias) { + UnboundAlias unboundAlias = (UnboundAlias) value; + return new Alias(TypeCoercionUtils.castUnbound(unboundAlias.child(), targetType)); } else { return TypeCoercionUtils.castUnbound(value, targetType); } @@ -484,8 +603,18 @@ private static NamedExpression generateDefaultExpression(Column column) { /** * get plan for explain. */ - public static Plan getPlanForExplain(ConnectContext ctx, LogicalPlan logicalQuery) { - return InsertUtils.normalizePlan(logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx), Optional.empty()); + public static Plan getPlanForExplain( + ConnectContext ctx, Optional analyzeContext, LogicalPlan logicalQuery) { + return InsertUtils.normalizePlan( + logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx), analyzeContext, Optional.empty()); + } + + /** supportFastInsertIntoValues */ + public static boolean supportFastInsertIntoValues( + LogicalPlan logicalPlan, TableIf targetTableIf, ConnectContext ctx) { + return logicalPlan instanceof UnboundTableSink && logicalPlan.child(0) instanceof InlineTable + && targetTableIf instanceof OlapTable + && ctx != null && ctx.getSessionVariable().isEnableFastAnalyzeInsertIntoValues(); } // check for insert into t1(a,b,gen_col) select 1,2,3; @@ -508,7 +637,7 @@ private static void checkGeneratedColumnForInsertIntoSelect(TableIf table, return; } Plan query = unboundLogicalSink.child(); - if (table instanceof OlapTable && !(query instanceof LogicalInlineTable)) { + if (table instanceof OlapTable && !(query instanceof InlineTable)) { OlapTable olapTable = (OlapTable) table; Set insertNames = Sets.newHashSet(); if (unboundLogicalSink.getColNames() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java index e7b1f4d581892c..0f3e320edcd4bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; @@ -93,8 +94,11 @@ protected static void analyzeGroupCommit(ConnectContext ctx, TableIf table, Logi conditions.add(Pair.of(() -> !(insertCtx.isPresent() && insertCtx.get() instanceof OlapInsertCommandContext && ((OlapInsertCommandContext) insertCtx.get()).isOverwrite()), () -> "is overwrite command")); conditions.add(Pair.of( - () -> tableSink.child() instanceof OneRowRelation || tableSink.child() instanceof LogicalUnion, - () -> "not one row relation or union, class: " + tableSink.child().getClass().getName())); + () -> tableSink.child() instanceof OneRowRelation + || tableSink.child() instanceof LogicalUnion + || tableSink.child() instanceof InlineTable, + () -> "not one row relation or union or inline table, class: " + + tableSink.child().getClass().getName())); ctx.setGroupCommit(conditions.stream().allMatch(p -> p.first.getAsBoolean())); if (!ctx.isGroupCommit() && LOG.isDebugEnabled()) { for (Pair> pair : conditions) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java index b2a2a1d83ca3e7..748bc8fdfa2223 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java @@ -17,15 +17,19 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; @@ -36,7 +40,7 @@ /** * represent value list such as values(1), (2), (3) will generate LogicalInlineTable((1), (2), (3)). */ -public class LogicalInlineTable extends LogicalLeaf implements BlockFuncDepsPropagation { +public class LogicalInlineTable extends LogicalLeaf implements InlineTable, BlockFuncDepsPropagation { private final List> constantExprsList; @@ -44,11 +48,16 @@ public LogicalInlineTable(List> constantExprsList) { this(constantExprsList, Optional.empty(), Optional.empty()); } + /** LogicalInlineTable */ public LogicalInlineTable(List> constantExprsList, Optional groupExpression, Optional logicalProperties) { super(PlanType.LOGICAL_INLINE_TABLE, groupExpression, logicalProperties); - this.constantExprsList = ImmutableList.copyOf( + + if (constantExprsList.isEmpty()) { + throw new AnalysisException("constantExprsList should now be empty"); + } + this.constantExprsList = Utils.fastToImmutableList( Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); } @@ -63,23 +72,49 @@ public R accept(PlanVisitor visitor, C context) { @Override public List getExpressions() { - return constantExprsList.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList()); + ImmutableList.Builder expressions = ImmutableList.builderWithExpectedSize( + constantExprsList.size() * constantExprsList.get(0).size()); + + for (List namedExpressions : constantExprsList) { + expressions.addAll(namedExpressions); + } + + return expressions.build(); } @Override public Plan withGroupExpression(Optional groupExpression) { - return null; + return new LogicalInlineTable( + constantExprsList, groupExpression, Optional.of(getLogicalProperties()) + ); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return null; + if (!children.isEmpty()) { + throw new AnalysisException("children should not be empty"); + } + return new LogicalInlineTable(constantExprsList, groupExpression, logicalProperties); } @Override public List computeOutput() { - return ImmutableList.of(); + int columnNum = constantExprsList.get(0).size(); + List firstRow = constantExprsList.get(0); + ImmutableList.Builder output = ImmutableList.builderWithExpectedSize(constantExprsList.size()); + for (int i = 0; i < columnNum; i++) { + NamedExpression firstRowColumn = firstRow.get(i); + boolean nullable = false; + for (List row : constantExprsList) { + if (row.get(i).nullable()) { + nullable = true; + break; + } + } + output.add(new SlotReference(firstRowColumn.getName(), firstRowColumn.getDataType(), nullable)); + } + return output.build(); } @Override @@ -98,4 +133,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(constantExprsList); } + + @Override + public String toString() { + return Utils.toSqlString("LogicalInlineTable[" + id.asInt() + "]", + "rowNum", constantExprsList.size(), + "constantExprsList", constantExprsList); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 7023815c7c5b99..9fa14458ed38b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -54,7 +54,7 @@ public LogicalOneRowRelation(RelationId relationId, List projec private LogicalOneRowRelation(RelationId relationId, List projects, Optional groupExpression, Optional logicalProperties) { super(relationId, PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); - this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); + this.projects = Utils.fastToImmutableList(Objects.requireNonNull(projects, "projects can not be null")); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 2e4ddb55ff2f02..e13ec2864b3cd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -216,7 +216,8 @@ public int getArity() { return children.size(); } - private DataType getAssignmentCompatibleType(DataType left, DataType right) { + /** getAssignmentCompatibleType */ + public static DataType getAssignmentCompatibleType(DataType left, DataType right) { if (left.isNullType()) { return right; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index 459044100b632d..d9fae844c48912 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; @@ -28,11 +29,14 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Union; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.BitSet; @@ -205,6 +209,14 @@ public void computeUniform(DataTrait.Builder builder) { // don't propagate uniform slots } + @Override + public boolean hasUnboundExpression() { + if (!constantExprsList.isEmpty() && children.isEmpty()) { + return false; + } + return super.hasUnboundExpression(); + } + private List mapSlotToIndex(Plan plan, List> equalSlotsList) { Map slotToIndex = new HashMap<>(); for (int i = 0; i < plan.getOutput().size(); i++) { @@ -280,4 +292,73 @@ public void computeEqualSet(DataTrait.Builder builder) { public void computeFd(DataTrait.Builder builder) { // don't generate } + + /** castCommonDataTypeAndNullableByConstants */ + public static Pair>, List> castCommonDataTypeAndNullableByConstants( + List> constantExprsList) { + int columnCount = constantExprsList.isEmpty() ? 0 : constantExprsList.get(0).size(); + Pair, List> commonInfo + = computeCommonDataTypeAndNullable(constantExprsList, columnCount); + List> castedRows = castToCommonType(constantExprsList, commonInfo.key(), columnCount); + List nullables = commonInfo.second; + return Pair.of(castedRows, nullables); + } + + private static Pair, List> computeCommonDataTypeAndNullable( + List> constantExprsList, int columnCount) { + List nullables = Lists.newArrayListWithCapacity(columnCount); + List commonDataTypes = Lists.newArrayListWithCapacity(columnCount); + List firstRow = constantExprsList.get(0); + for (int columnId = 0; columnId < columnCount; columnId++) { + Expression constant = firstRow.get(columnId).child(0); + Pair commonDataTypeAndNullable + = computeCommonDataTypeAndNullable(constant, columnId, constantExprsList); + commonDataTypes.add(commonDataTypeAndNullable.first); + nullables.add(commonDataTypeAndNullable.second); + } + return Pair.of(commonDataTypes, nullables); + } + + private static Pair computeCommonDataTypeAndNullable( + Expression firstRowExpr, int columnId, List> constantExprsList) { + DataType commonDataType = firstRowExpr.getDataType(); + boolean nullable = firstRowExpr.nullable(); + for (int rowId = 1; rowId < constantExprsList.size(); rowId++) { + NamedExpression namedExpression = constantExprsList.get(rowId).get(columnId); + Expression otherConstant = namedExpression.child(0); + nullable |= otherConstant.nullable(); + DataType otherDataType = otherConstant.getDataType(); + commonDataType = getAssignmentCompatibleType(commonDataType, otherDataType); + } + return Pair.of(commonDataType, nullable); + } + + private static List> castToCommonType( + List> rows, List commonDataTypes, int columnCount) { + ImmutableList.Builder> castedConstants + = ImmutableList.builderWithExpectedSize(rows.size()); + for (List row : rows) { + castedConstants.add(castToCommonType(row, commonDataTypes)); + } + return castedConstants.build(); + } + + private static List castToCommonType(List row, List commonTypes) { + ImmutableList.Builder castedRow = ImmutableList.builderWithExpectedSize(row.size()); + boolean changed = false; + for (int columnId = 0; columnId < row.size(); columnId++) { + NamedExpression constantAlias = row.get(columnId); + Expression constant = constantAlias.child(0); + DataType commonType = commonTypes.get(columnId); + if (commonType.equals(constant.getDataType())) { + castedRow.add(constantAlias); + } else { + changed = true; + Expression expression + = TypeCoercionUtils.castIfNotSameTypeStrict(constant, commonType); + castedRow.add((NamedExpression) constantAlias.withChildren(expression)); + } + } + return changed ? castedRow.build() : row; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java index efe0a03e3708b4..90e92ca1ae2597 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java @@ -178,7 +178,7 @@ public int hashCode() { @Override public String toString() { - return Utils.toSqlString("LogicalOlapTableSink[" + id.asInt() + "]", + return Utils.toSqlString("PhysicalOlapTableSink[" + id.asInt() + "]", "outputExprs", outputExprs, "database", database.getFullName(), "targetTable", targetTable.getName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java index 396c6e4f26569f..f7642ce572c1a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.visitor; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.Command; @@ -99,6 +100,7 @@ public R visitCommand(Command command, C context) { return visit(command, context); } + // ******************************* // relations // ******************************* @@ -130,6 +132,10 @@ public R visitPhysicalSink(PhysicalSink physicalSink, C context) // ******************************* // Logical plans // ******************************* + public R visitUnboundInlineTable(UnboundInlineTable unboundInlineTable, C context) { + return visit(unboundInlineTable, context); + } + public R visitLogicalSqlCache(LogicalSqlCache sqlCache, C context) { return visit(sqlCache, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 1da4353d20da33..1fb108b79fd6a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -601,8 +601,7 @@ public static Optional characterLiteralTypeCoercion(String value, Da } else if (dataType.isDateTimeType() && DateTimeChecker.isValidDateTime(value)) { ret = DateTimeLiteral.parseDateTimeLiteral(value, false).orElse(null); } else if (dataType.isDateV2Type() && DateTimeChecker.isValidDateTime(value)) { - Result parseResult - = DateV2Literal.parseDateLiteral(value); + Result parseResult = DateV2Literal.parseDateLiteral(value); if (parseResult.isOk()) { ret = parseResult.get(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index cd1be6c5cb6df7..446960f9d56415 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -37,9 +37,9 @@ import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.plugin.AuditEvent.AuditEventBuilder; @@ -162,8 +162,8 @@ private static int countValues(List children) { for (Plan child : children) { if (child instanceof UnboundOneRowRelation) { cnt++; - } else if (child instanceof LogicalInlineTable) { - cnt += ((LogicalInlineTable) child).getConstantExprsList().size(); + } else if (child instanceof InlineTable) { + cnt += ((InlineTable) child).getConstantExprsList().size(); } else if (child instanceof LogicalUnion) { cnt += countValues(child.children()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index cf26cce7383e1c..d62c00534b6d94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -366,6 +366,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_SINGLE_REPLICA_INSERT = "enable_single_replica_insert"; + public static final String ENABLE_FAST_ANALYZE_INSERT_INTO_VALUES = "enable_fast_analyze_into_values"; + public static final String ENABLE_FUNCTION_PUSHDOWN = "enable_function_pushdown"; public static final String ENABLE_EXT_FUNC_PRED_PUSHDOWN = "enable_ext_func_pred_pushdown"; @@ -1497,6 +1499,15 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { needForward = true, varType = VariableAnnotation.EXPERIMENTAL) public boolean enableSingleReplicaInsert = false; + @VariableMgr.VarAttr( + name = ENABLE_FAST_ANALYZE_INSERT_INTO_VALUES, fuzzy = true, + description = { + "跳过大部分的优化规则，快速分析insert into values语句", + "Skip most optimization rules and quickly analyze insert into values statements" + } + ) + private boolean enableFastAnalyzeInsertIntoValues = true; + @VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN, fuzzy = true) public boolean enableFunctionPushdown = false; @@ -3654,8 +3665,6 @@ public boolean isEnableExprTrace() { return enableExprTrace; } - - public boolean isEnableSingleReplicaInsert() { return enableSingleReplicaInsert; } @@ -3664,6 +3673,14 @@ public void setEnableSingleReplicaInsert(boolean enableSingleReplicaInsert) { this.enableSingleReplicaInsert = enableSingleReplicaInsert; } + public boolean isEnableFastAnalyzeInsertIntoValues() { + return enableFastAnalyzeInsertIntoValues; + } + + public void setEnableFastAnalyzeInsertIntoValues(boolean enableFastAnalyzeInsertIntoValues) { + this.enableFastAnalyzeInsertIntoValues = enableFastAnalyzeInsertIntoValues; + } + public boolean isEnableMemtableOnSinkNode() { return enableMemtableOnSinkNode; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java index 29889efdd6ce13..32b3706b290fbd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java @@ -17,14 +17,12 @@ package org.apache.doris.nereids.rules.expression.rules; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; class SimplifyComparisonPredicateSqlTest extends TestWithFeService implements MemoPatternMatchSupported { @@ -153,17 +151,30 @@ void dateLikeOverflow() { ) ); - Assertions.assertThrows(AnalysisException.class, () -> PlanChecker.from(connectContext) + PlanChecker.from(connectContext) .analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '2021-01-32 00:00:00'") .rewrite() - ); - Assertions.assertThrows(AnalysisException.class, () -> PlanChecker.from(connectContext) + .matches(logicalOneRowRelation().when(oneRowRelation -> + oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral) + ); + + PlanChecker.from(connectContext) + .analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '2021-01-32 00:00:00'") + .rewrite() + .matches(logicalOneRowRelation().when(oneRowRelation -> + oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral) + ); + PlanChecker.from(connectContext) .analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '2021-01-32 23:00:00'") .rewrite() - ); - Assertions.assertThrows(AnalysisException.class, () -> PlanChecker.from(connectContext) + .matches(logicalOneRowRelation().when(oneRowRelation -> + oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral) + ); + PlanChecker.from(connectContext) .analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '1000'") .rewrite() - ); + .matches(logicalOneRowRelation().when(oneRowRelation -> + oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral) + ); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java index a002fe1a9c04a9..8279cd5cc4ffeb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java @@ -136,7 +136,7 @@ public void testParse() { )); // need select * - String sql3 = "seelct k1, k2, v1, v2 replace(k1 / 2 as k1) from t1"; + String sql3 = "select k1, k2, v1, v2 replace(k1 / 2 as k1) from t1"; Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()) .checkParse(sql3, (checker) -> checker.matches( logicalProject( @@ -152,7 +152,7 @@ public void testParse() { .checkParse(sql4, (checker) -> checker.matches( logicalProject( logicalCheckPolicy( - unboundRelation() + logicalOneRowRelation() ) ) ))); @@ -162,9 +162,7 @@ public void testParse() { Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()) .checkParse(sql5, (checker) -> checker.matches( logicalProject( - logicalCheckPolicy( - unboundRelation() - ) + logicalOneRowRelation() ) ))); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java index 8db1c9446d0c6d..786355c83b76c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.time.DateTimeException; import java.util.function.Consumer; class DateLiteralTest { @@ -67,7 +68,7 @@ void testDate() { new DateLiteral("2022-1-1"); new DateLiteral("20220101"); - Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("-01-01")); + Assertions.assertThrows(DateTimeException.class, () -> new DateLiteral("-01-01")); } @Test @@ -128,8 +129,8 @@ void testIrregularDate() { @Test void testWrongPunctuationDate() { - Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020€02€01")); - Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020【02】01")); + Assertions.assertThrows(DateTimeException.class, () -> new DateTimeV2Literal("2020€02€01")); + Assertions.assertThrows(DateTimeException.class, () -> new DateTimeV2Literal("2020【02】01")); } @Test diff --git a/regression-test/suites/compression_p0/load.groovy b/regression-test/suites/compression_p0/load.groovy index 70eeafea12ae48..722732f65c3b06 100644 --- a/regression-test/suites/compression_p0/load.groovy +++ b/regression-test/suites/compression_p0/load.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_compression", "p0") { +suite("load") { // test snappy compression algorithm def tableName = "test_snappy" diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy index 633ad98d86f556..b448ad406bbde2 100644 --- a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy +++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy @@ -32,7 +32,7 @@ suite("test_nestedtypes_insert_into_select", "p0") { test { sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" - exception "mismatched input 'named_struct' expecting" + exception "no viable alternative at input '[named_struct'" } @@ -50,6 +50,6 @@ suite("test_nestedtypes_insert_into_select", "p0") { test { sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" - exception "mismatched input 'named_struct' expecting" + exception "no viable alternative at input '[named_struct'" } } diff --git a/regression-test/suites/index_p0/load.groovy b/regression-test/suites/index_p0/load.groovy index 174339f148300a..5416a5096329cb 100644 --- a/regression-test/suites/index_p0/load.groovy +++ b/regression-test/suites/index_p0/load.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_bitmap_index_load") { +suite("load") { def tbName = "test_decimal_bitmap_index_multi_page" sql """ diff --git a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy index 054add11d9f3a3..166d329c455511 100644 --- a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy +++ b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy @@ -241,7 +241,7 @@ suite("insert_group_commit_with_exception") { assertTrue(false) } catch (Exception e) { logger.info("exception : " + e) - assertTrue(e.getMessage().contains("insert into cols should be corresponding to the query output")) + assertTrue(e.getMessage().contains("Column count doesn't match value count")) } } getRowCount(14) diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy index a615c7316bdb56..44fc259a71a1cc 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy @@ -19,7 +19,7 @@ This suite is a two dimensional test case file. It mainly tests the inner join and filter positions. */ -suite("partition_mv_rewrite_dimension_2_2") { +suite("dimension_2_inner_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}"