Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[enhancement](nereids) improve lots of values in insert into values statement #40202

Merged
merged 17 commits into from
Dec 23, 2024
3 changes: 2 additions & 1 deletion be/src/http/http_channel.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ void HttpChannel::send_files(HttpRequest* request, const std::string& root_dir,
VLOG_DEBUG << "http channel send file " << file_path << ", size: " << file_size;

evbuffer_add_printf(evb.get(), "File-Name: %s\r\n", file.c_str());
evbuffer_add_printf(evb.get(), "Content-Length: %ld\r\n", file_size);
evbuffer_add_printf(evb.get(), "Content-Length: %" PRIi64 "\r\n", file_size);
morrySnow marked this conversation as resolved.
Show resolved Hide resolved

evbuffer_add_printf(evb.get(), "\r\n");
if (file_size > 0) {
evbuffer_add_file(evb.get(), fd, 0, file_size);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1476,7 +1476,9 @@ rowConstructor
;

rowConstructorItem
: namedExpression | DEFAULT
: constant // duplicate constant rule for improve the parse of `insert into tbl values`
| DEFAULT
| namedExpression
;

predicate
Expand Down Expand Up @@ -1678,7 +1680,7 @@ constant
| LEFT_BRACE (items+=constant COLON items+=constant)?
(COMMA items+=constant COLON items+=constant)* RIGHT_BRACE #mapLiteral
| LEFT_BRACE items+=constant (COMMA items+=constant)* RIGHT_BRACE #structLiteral
| PLACEHOLDER #placeholder
| PLACEHOLDER #placeholder
;

comparisonOperator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -473,9 +473,16 @@ public void setCurrentRootRewriteJobContext(RootRewriteJobContext currentRootRew
this.currentRootRewriteJobContext = Optional.ofNullable(currentRootRewriteJobContext);
}

/** showPlanProcess */
public boolean showPlanProcess() {
Boolean show = showPlanProcess.get();
return show != null && show;
if (show != null && show) {
return true;
}
if (parent.isPresent()) {
return parent.get().showPlanProcess();
}
return false;
}

/** set showPlanProcess in task scope */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,19 @@
*/
public class NereidsPlanner extends Planner {
public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class);

protected Plan parsedPlan;
protected Plan analyzedPlan;
protected Plan rewrittenPlan;
protected Plan optimizedPlan;
protected PhysicalPlan physicalPlan;

private CascadesContext cascadesContext;
private final StatementContext statementContext;
private final List<ScanNode> scanNodeList = Lists.newArrayList();
private final List<PhysicalRelation> physicalRelations = Lists.newArrayList();
private DescriptorTable descTable;

private Plan parsedPlan;
private Plan analyzedPlan;
private Plan rewrittenPlan;
private Plan optimizedPlan;
private PhysicalPlan physicalPlan;
private FragmentIdMapping<DistributedPlan> distributedPlans;
// The cost of optimized plan
private double cost = 0;
Expand Down Expand Up @@ -552,7 +554,7 @@ public Group getRoot() {
return cascadesContext.getMemo().getRoot();
}

private PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) {
protected PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) {
if (nthPlan <= 1) {
cost = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow(
() -> new AnalysisException("lowestCostPlans with physicalProperties("
Expand Down Expand Up @@ -605,6 +607,9 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical
}

private long getGarbageCollectionTime() {
if (!ConnectContext.get().getSessionVariable().enableProfile()) {
morrySnow marked this conversation as resolved.
Show resolved Hide resolved
return 0;
}
List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
long initialGCTime = 0;
for (GarbageCollectorMXBean gcBean : gcMxBeans) {
Expand Down Expand Up @@ -881,7 +886,7 @@ private boolean showPlanProcess(ExplainOptions explainOptions) {
return explainOptions != null && explainOptions.showPlanProcess();
}

private void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) {
protected void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) {
if (showPlanProcess) {
cascadesContext.withPlanProcess(showPlanProcess, task);
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.InlineTable;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;

import com.google.common.collect.ImmutableList;

import java.util.List;
import java.util.Objects;
import java.util.Optional;

/** UnboundInlineTable */
public class UnboundInlineTable extends LogicalLeaf implements InlineTable, BlockFuncDepsPropagation, UnboundPlan {
private final List<List<NamedExpression>> constantExprsList;

public UnboundInlineTable(List<List<NamedExpression>> constantExprsList) {
super(PlanType.LOGICAL_UNBOUND_INLINE_TABLE, Optional.empty(), Optional.empty());
this.constantExprsList = Utils.fastToImmutableList(
Objects.requireNonNull(constantExprsList, "constantExprsList can not be null")
);
}

public List<List<NamedExpression>> getConstantExprsList() {
return constantExprsList;
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitUnboundInlineTable(this, context);
}

@Override
public List<? extends Expression> getExpressions() {
ImmutableList.Builder<Expression> expressions = ImmutableList.builderWithExpectedSize(
constantExprsList.size() * constantExprsList.get(0).size());

for (List<NamedExpression> namedExpressions : constantExprsList) {
expressions.addAll(namedExpressions);
}

return expressions.build();
}

@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return this;
}

@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return this;
}

@Override
public List<Slot> computeOutput() {
throw new UnboundException("output");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.analyzer;

import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.UnboundLogicalProperties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;

import java.util.List;

/** UnboundPlan */
public interface UnboundPlan extends Plan {
@Override
default LogicalProperties computeLogicalProperties() {
return UnboundLogicalProperties.INSTANCE;
}

@Override
default List<Slot> computeOutput() {
throw new UnboundException("output");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import org.apache.commons.lang3.StringUtils;

import java.util.List;
import java.util.Objects;
Expand Down Expand Up @@ -176,4 +177,12 @@ public LogicalProperties computeLogicalProperties() {
public List<Slot> computeOutput() {
throw new UnboundException("output");
}

@Override
public String toString() {
return Utils.toSqlString("UnboundTableSink",
"nameParts", StringUtils.join(nameParts, "."),
"colNames", colNames,
"hints", hints);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
Expand Down Expand Up @@ -2058,17 +2059,22 @@ public PlanFragment visitPhysicalSetOperation(
}
setOperationNode.setNereidsId(setOperation.getId());

setOperation.getRegularChildrenOutputs().stream()
.map(o -> o.stream()
.map(e -> ExpressionTranslator.translate(e, context))
.collect(ImmutableList.toImmutableList()))
.forEach(setOperationNode::addResultExprLists);
for (List<SlotReference> regularChildrenOutput : setOperation.getRegularChildrenOutputs()) {
Builder<Expr> translateOutputs = ImmutableList.builderWithExpectedSize(regularChildrenOutput.size());
for (SlotReference childOutput : regularChildrenOutput) {
translateOutputs.add(ExpressionTranslator.translate(childOutput, context));
}
setOperationNode.addResultExprLists(translateOutputs.build());
}

if (setOperation instanceof PhysicalUnion) {
((PhysicalUnion) setOperation).getConstantExprsList().stream()
.map(l -> l.stream()
.map(e -> ExpressionTranslator.translate(e, context))
.collect(ImmutableList.toImmutableList()))
.forEach(setOperationNode::addConstExprList);
for (List<NamedExpression> unionConsts : ((PhysicalUnion) setOperation).getConstantExprsList()) {
Builder<Expr> translateConsts = ImmutableList.builderWithExpectedSize(unionConsts.size());
for (NamedExpression unionConst : unionConsts) {
translateConsts.add(ExpressionTranslator.translate(unionConst, context));
}
setOperationNode.addConstExprList(translateConsts.build());
}
}

for (PlanFragment childFragment : childrenFragments) {
Expand Down
Loading
Loading