Skip to content

Commit

Permalink
optimize insert into values
Browse files Browse the repository at this point in the history
  • Loading branch information
@924060929
924060929 committed Dec 9, 2024
1 parent a25d457 commit 5c8f0a7
Show file tree
Hide file tree
Showing 14 changed files with 495 additions and 92 deletions.
14 changes: 8 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,17 +99,19 @@
*/
public class NereidsPlanner extends Planner {
public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class);

protected Plan parsedPlan;
protected Plan analyzedPlan;
protected Plan rewrittenPlan;
protected Plan optimizedPlan;
protected PhysicalPlan physicalPlan;

private CascadesContext cascadesContext;
private final StatementContext statementContext;
private final List<ScanNode> scanNodeList = Lists.newArrayList();
private final List<PhysicalRelation> physicalRelations = Lists.newArrayList();
private DescriptorTable descTable;

private Plan parsedPlan;
private Plan analyzedPlan;
private Plan rewrittenPlan;
private Plan optimizedPlan;
private PhysicalPlan physicalPlan;
private FragmentIdMapping<DistributedPlan> distributedPlans;
// The cost of optimized plan
private double cost = 0;
Expand Down Expand Up @@ -540,7 +542,7 @@ public Group getRoot() {
return cascadesContext.getMemo().getRoot();
}

private PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) {
protected PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) {
if (nthPlan <= 1) {
cost = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow(
() -> new AnalysisException("lowestCostPlans with physicalProperties("
Expand Down
26 changes: 16 additions & 10 deletions ...e-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
Expand Down Expand Up @@ -2056,17 +2057,22 @@ public PlanFragment visitPhysicalSetOperation(
}
setOperationNode.setNereidsId(setOperation.getId());

setOperation.getRegularChildrenOutputs().stream()
.map(o -> o.stream()
.map(e -> ExpressionTranslator.translate(e, context))
.collect(ImmutableList.toImmutableList()))
.forEach(setOperationNode::addResultExprLists);
for (List<SlotReference> regularChildrenOutput : setOperation.getRegularChildrenOutputs()) {
Builder<Expr> translateOutputs = ImmutableList.builderWithExpectedSize(regularChildrenOutput.size());
for (SlotReference childOutput : regularChildrenOutput) {
translateOutputs.add(ExpressionTranslator.translate(childOutput, context));
}
setOperationNode.addResultExprLists(translateOutputs.build());
}

if (setOperation instanceof PhysicalUnion) {
((PhysicalUnion) setOperation).getConstantExprsList().stream()
.map(l -> l.stream()
.map(e -> ExpressionTranslator.translate(e, context))
.collect(ImmutableList.toImmutableList()))
.forEach(setOperationNode::addConstExprList);
for (List<NamedExpression> unionConsts : ((PhysicalUnion) setOperation).getConstantExprsList()) {
Builder<Expr> translateConsts = ImmutableList.builderWithExpectedSize(unionConsts.size());
for (NamedExpression unionConst : unionConsts) {
translateConsts.add(ExpressionTranslator.translate(unionConst, context));
}
setOperationNode.addConstExprList(translateConsts.build());
}
}

for (PlanFragment childFragment : childrenFragments) {
Expand Down
21 changes: 14 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1838,10 +1838,13 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte

@Override
public LogicalPlan visitInlineTable(InlineTableContext ctx) {
List<List<NamedExpression>> values = ctx.rowConstructor().stream()
.map(this::visitRowConstructor)
.collect(ImmutableList.toImmutableList());
return new LogicalInlineTable(values);
List<RowConstructorContext> rowConstructorContexts = ctx.rowConstructor();
ImmutableList.Builder<List<NamedExpression>> rows
= ImmutableList.builderWithExpectedSize(rowConstructorContexts.size());
for (RowConstructorContext rowConstructorContext : rowConstructorContexts) {
rows.add(visitRowConstructor(rowConstructorContext));
}
return new LogicalInlineTable(rows.build());
}

/**
Expand Down Expand Up @@ -2972,9 +2975,13 @@ public Expression visitParenthesizedExpression(ParenthesizedExpressionContext ct

@Override
public List<NamedExpression> visitRowConstructor(RowConstructorContext ctx) {
return ctx.rowConstructorItem().stream()
.map(this::visitRowConstructorItem)
.collect(ImmutableList.toImmutableList());
List<RowConstructorItemContext> rowConstructorItemContexts = ctx.rowConstructorItem();
ImmutableList.Builder<NamedExpression> columns
= ImmutableList.builderWithExpectedSize(rowConstructorItemContexts.size());
for (RowConstructorItemContext rowConstructorItemContext : rowConstructorItemContexts) {
columns.add(visitRowConstructorItem(rowConstructorItemContext));
}
return columns.build();
}

@Override
Expand Down
49 changes: 31 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.logging.log4j.Logger;

import java.util.BitSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -273,37 +274,40 @@ private <T> T parse(String sql, Function<DorisParser, ParserRuleContext> parseFu

private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
Function<DorisParser, ParserRuleContext> parseFunction) {
ParserRuleContext tree = toAst(sql, parseFunction);
CommonTokenStream tokenStream = parseAllTokens(sql);
ParserRuleContext tree = toAst(tokenStream, parseFunction);
LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null
? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder;
? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint))
: logicalPlanBuilder;
return (T) realLogicalPlanBuilder.visit(tree);
}

public LogicalPlan parseForCreateView(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
CommonTokenStream tokenStream = parseAllTokens(sql);
ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView(
getHintMap(sql, DorisParser::selectHint));
getHintMap(sql, tokenStream, DorisParser::selectHint));
return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
}

/** parseForSyncMv */
public Optional<String> parseForSyncMv(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
CommonTokenStream tokenStream = parseAllTokens(sql);
ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
getHintMap(sql, DorisParser::selectHint));
getHintMap(sql, tokenStream, DorisParser::selectHint));
logicalPlanBuilderForSyncMv.visit(tree);
return logicalPlanBuilderForSyncMv.getQuerySql();
}

/** get hint map */
public static Map<Integer, ParserRuleContext> getHintMap(String sql,
public static Map<Integer, ParserRuleContext> getHintMap(String sql, CommonTokenStream hintTokenStream,
Function<DorisParser, ParserRuleContext> parseFunction) {
// parse hint first round
DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer);

Map<Integer, ParserRuleContext> selectHintMap = Maps.newHashMap();

Token hintToken = hintTokenStream.getTokenSource().nextToken();
Iterator<Token> tokenIterator = hintTokenStream.getTokens().iterator();
Token hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null;
while (hintToken != null && hintToken.getType() != DorisLexer.EOF) {
if (hintToken.getChannel() == 2 && sql.charAt(hintToken.getStartIndex() + 2) == '+') {
String hintSql = sql.substring(hintToken.getStartIndex() + 3, hintToken.getStopIndex() + 1);
Expand All @@ -313,15 +317,19 @@ public static Map<Integer, ParserRuleContext> getHintMap(String sql,
ParserRuleContext hintContext = parseFunction.apply(hintParser);
selectHintMap.put(hintToken.getStartIndex(), hintContext);
}
hintToken = hintTokenStream.getTokenSource().nextToken();
hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null;
}
return selectHintMap;
}

public static ParserRuleContext toAst(
String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
return toAst(parseAllTokens(sql), parseFunction);
}

/** toAst */
public static ParserRuleContext toAst(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
public static ParserRuleContext toAst(
CommonTokenStream tokenStream, Function<DorisParser, ParserRuleContext> parseFunction) {
DorisParser parser = new DorisParser(tokenStream);

parser.addParseListener(POST_PROCESSOR);
Expand Down Expand Up @@ -352,9 +360,7 @@ public static ParserRuleContext toAst(String sql, Function<DorisParser, ParserRu
* will be normalized to: select \/*+SET_VAR(key=value)*\/ * , a, b from table
*/
public static String removeCommentAndTrimBlank(String sql) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
CommonTokenStream tokenStream = parseAllTokens(sql);

// maybe add more space char
StringBuilder newSql = new StringBuilder((int) (sql.length() * 1.2));
Expand All @@ -381,4 +387,11 @@ public static String removeCommentAndTrimBlank(String sql) {
}
return newSql.toString().trim();
}

private static CommonTokenStream parseAllTokens(String sql) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
return tokenStream;
}
}
9 changes: 9 additions & 0 deletions ...e/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.analyzer.UnboundVariable;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule;
import org.apache.doris.nereids.rules.expression.ExpressionListenerMatcher;
import org.apache.doris.nereids.rules.expression.ExpressionMatchingContext;
Expand Down Expand Up @@ -53,6 +55,7 @@
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
import org.apache.doris.nereids.trees.expressions.Variable;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
Expand Down Expand Up @@ -220,6 +223,12 @@ public Expression visitMatch(Match match, ExpressionRewriteContext context) {
return super.visitMatch(match, context);
}

@Override
public Expression visitUnboundVariable(UnboundVariable unboundVariable, ExpressionRewriteContext context) {
Variable variable = ExpressionAnalyzer.resolveUnboundVariable(unboundVariable);
return variable.getRealExpression();
}

@Override
public Expression visitEncryptKeyRef(EncryptKeyRef encryptKeyRef, ExpressionRewriteContext context) {
String dbName = encryptKeyRef.getDbName();
Expand Down
38 changes: 30 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,16 +287,23 @@ public static Result<DateLiteral, AnalysisException> parseDateLiteral(String s)

/** parseDateTime */
public static Result<TemporalAccessor, AnalysisException> parseDateTime(String s) {
// fast parse '2022-01-01'
if (s.length() == 10 && s.charAt(4) == '-' && s.charAt(7) == '-') {
TemporalAccessor date = fastParseDate(s);
if (date != null) {
return Result.ok(date);
}
}

String originalString = s;
try {
// fast parse '2022-01-01'
if ((s.length() == 10 || s.length() == 19) && s.charAt(4) == '-' && s.charAt(7) == '-') {
if (s.length() == 10) {
TemporalAccessor date = fastParseDate(s);
if (date != null) {
return Result.ok(date);
}
} else if (s.charAt(10) == ' ' && s.charAt(13) == ':' && s.charAt(16) == ':') {
TemporalAccessor date = fastParseDateTime(s);
if (date != null) {
return Result.ok(date);
}
}
}

TemporalAccessor dateTime;

// remove suffix/prefix ' '
Expand Down Expand Up @@ -566,6 +573,21 @@ private static TemporalAccessor fastParseDate(String date) {
}
}

private static TemporalAccessor fastParseDateTime(String date) {
Integer year = readNextInt(date, 0, 4);
Integer month = readNextInt(date, 5, 2);
Integer day = readNextInt(date, 8, 2);
Integer hour = readNextInt(date, 11, 2);
Integer minute = readNextInt(date, 14, 2);
Integer second = readNextInt(date, 17, 2);

if (year != null && month != null && day != null && hour != null && minute != null && second != null) {
return LocalDateTime.of(year, month, day, hour, minute, second);
} else {
return null;
}
}

private static Integer readNextInt(String str, int offset, int readLength) {
int value = 0;
int realReadLength = 0;
Expand Down
6 changes: 4 additions & 2 deletions ...ava/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public BatchInsertIntoTableCommand(LogicalPlan logicalQuery) {

@Override
public Plan getExplainPlan(ConnectContext ctx) throws Exception {
return InsertUtils.getPlanForExplain(ctx, this.logicalQuery);
return InsertUtils.getPlanForExplain(ctx, Optional.empty(), this.logicalQuery);
}

@Override
Expand All @@ -98,7 +98,9 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx);
targetTableIf.readLock();
try {
this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty());
this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(
logicalQuery, targetTableIf, Optional.empty(), Optional.empty()
);
LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext());
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
Expand Down
Loading

0 comments on commit 5c8f0a7

Please sign in to comment.