Skip to content

Commit

Permalink
fix(interactive): Fix Concurrent Bugs in Graph Optimizer (#4269)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shirly121
shirly121 authored Sep 29, 2024
1 parent 1b4b809 commit 03fe273
Show file tree
Hide file tree
Showing 6 changed files with 344 additions and 166 deletions.
12 changes: 12 additions & 0 deletions ...ive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/PlannerConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ public class PlannerConfig {
// in Neo4j
public static final Config<Boolean> JOIN_BY_EDGE_ENABLED =
Config.boolConfig("graph.planner.join.by.edge.enabled", false);
public static final Config<Integer> GRAPH_PLANNER_GROUP_SIZE =
Config.intConfig("graph.planner.group.size", 8);
public static final Config<Integer> GRAPH_PLANNER_GROUP_CLEAR_INTERVAL_MINUTES =
Config.intConfig("graph.planner.group.clear.interval.minutes", 30);

private final Configs configs;
private final List<String> rules;
Expand Down Expand Up @@ -96,6 +100,14 @@ public String getJoinByForeignKeyUri() {
return GraphConfig.GRAPH_FOREIGN_KEY_URI.get(configs);
}

public int getPlannerGroupSize() {
return GRAPH_PLANNER_GROUP_SIZE.get(configs);
}

public int getPlannerGroupClearIntervalMinutes() {
return GRAPH_PLANNER_GROUP_CLEAR_INTERVAL_MINUTES.get(configs);
}

@Override
public String toString() {
return "PlannerConfig{"
Expand Down
191 changes: 27 additions & 164 deletions ...ne/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,6 @@
import com.alibaba.graphscope.common.ir.meta.IrMeta;
import com.alibaba.graphscope.common.ir.meta.glogue.calcite.GraphRelMetadataQuery;
import com.alibaba.graphscope.common.ir.meta.glogue.calcite.handler.GraphMetadataHandlerProvider;
import com.alibaba.graphscope.common.ir.meta.schema.foreign.ForeignKeyMeta;
import com.alibaba.graphscope.common.ir.planner.rules.*;
import com.alibaba.graphscope.common.ir.planner.volcano.VolcanoPlannerX;
import com.alibaba.graphscope.common.ir.rel.GraphShuttle;
import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource;
import com.alibaba.graphscope.common.ir.rel.graph.match.AbstractLogicalMatch;
Expand All @@ -36,19 +33,14 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;

import org.apache.calcite.plan.*;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgram;
import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.plan.GraphOptCluster;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rel.rules.FilterJoinRule;
import org.apache.calcite.tools.RelBuilderFactory;
import org.checkerframework.checker.nullness.qual.Nullable;

Expand All @@ -59,75 +51,59 @@
* Optimize graph relational tree which consists of match and other relational operators
*/
public class GraphRelOptimizer {
private final Configs graphConfig;
private final PlannerConfig config;
private final RelOptPlanner relPlanner;
private final RelOptPlanner matchPlanner;
private final RelOptPlanner physicalPlanner;
private final RelBuilderFactory relBuilderFactory;
private final GlogueHolder glogueHolder;
private final PlannerGroupManager plannerGroupManager;

public GraphRelOptimizer(Configs graphConfig, Class<? extends PlannerGroupManager> instance) {
try {
this.config = new PlannerConfig(graphConfig);
this.relBuilderFactory = new GraphBuilderFactory(graphConfig);
this.glogueHolder = new GlogueHolder(graphConfig);
this.plannerGroupManager =
instance.getDeclaredConstructor(PlannerConfig.class, RelBuilderFactory.class)
.newInstance(this.config, this.relBuilderFactory);
} catch (Exception e) {
throw new RuntimeException(e);
}
}

public GraphRelOptimizer(Configs graphConfig) {
this.graphConfig = graphConfig;
this.config = new PlannerConfig(graphConfig);
this.relBuilderFactory = new GraphBuilderFactory(graphConfig);
this.relPlanner = createRelPlanner();
this.matchPlanner = createMatchPlanner();
this.physicalPlanner = createPhysicalPlanner();
this.glogueHolder = new GlogueHolder(graphConfig);
this(graphConfig, PlannerGroupManager.Dynamic.class);
}

public GlogueHolder getGlogueHolder() {
return glogueHolder;
}

public RelOptPlanner getMatchPlanner() {
return matchPlanner;
}

public RelOptPlanner getPhysicalPlanner() {
return physicalPlanner;
PlannerGroup currentGroup = this.plannerGroupManager.getCurrentGroup();
return currentGroup.getMatchPlanner();
}

public RelOptPlanner getRelPlanner() {
return relPlanner;
public RelNode optimize(RelNode before, GraphIOProcessor ioProcessor) {
PlannerGroup currentGroup = this.plannerGroupManager.getCurrentGroup();
return currentGroup.optimize(before, ioProcessor);
}

public @Nullable RelMetadataQuery createMetaDataQuery(IrMeta irMeta) {
if (config.isOn() && config.getOpt() == PlannerConfig.Opt.CBO) {
GlogueQuery gq = this.glogueHolder.getGlogue();
Preconditions.checkArgument(gq != null, "glogue is not ready");
return new GraphRelMetadataQuery(
new GraphMetadataHandlerProvider(this.matchPlanner, gq, this.config));
new GraphMetadataHandlerProvider(getMatchPlanner(), gq, this.config));
}
return null;
}

public RelNode optimize(RelNode before, GraphIOProcessor ioProcessor) {
if (config.isOn()) {
// apply rules of 'FilterPushDown' before the match optimization
relPlanner.setRoot(before);
RelNode relOptimized = relPlanner.findBestExp();
if (config.getOpt() == PlannerConfig.Opt.CBO) {
relOptimized = relOptimized.accept(new MatchOptimizer(ioProcessor));
}
// apply rules of 'FieldTrim' after the match optimization
if (config.getRules().contains(FieldTrimRule.class.getSimpleName())) {
relOptimized = FieldTrimRule.trim(ioProcessor.getBuilder(), relOptimized);
}
physicalPlanner.setRoot(relOptimized);
RelNode physicalOptimized = physicalPlanner.findBestExp();
clear();
return physicalOptimized;
}
return before;
}

private class MatchOptimizer extends GraphShuttle {
public static class MatchOptimizer extends GraphShuttle {
private final GraphIOProcessor ioProcessor;
private final RelOptPlanner matchPlanner;

public MatchOptimizer(GraphIOProcessor ioProcessor) {
public MatchOptimizer(GraphIOProcessor ioProcessor, RelOptPlanner matchPlanner) {
this.ioProcessor = ioProcessor;
this.matchPlanner = matchPlanner;
}

@Override
Expand Down Expand Up @@ -219,117 +195,4 @@ public void visit(RelNode node, int ordinal, @Nullable RelNode parent) {
return decomposable.get();
}
}

private RelOptPlanner createRelPlanner() {
HepProgramBuilder hepBuilder = HepProgram.builder();
if (config.isOn()) {
List<RelRule.Config> ruleConfigs = Lists.newArrayList();
config.getRules()
.forEach(
k -> {
if (k.equals(
FilterJoinRule.FilterIntoJoinRule.class.getSimpleName())) {
ruleConfigs.add(CoreRules.FILTER_INTO_JOIN.config);
} else if (k.equals(FilterMatchRule.class.getSimpleName())) {
ruleConfigs.add(FilterMatchRule.Config.DEFAULT);
}
});
ruleConfigs.forEach(
k -> {
hepBuilder.addRuleInstance(
k.withRelBuilderFactory(relBuilderFactory).toRule());
});
}
return new HepPlanner(hepBuilder.build());
}

private RelOptPlanner createMatchPlanner() {
if (config.isOn() && config.getOpt() == PlannerConfig.Opt.CBO) {
VolcanoPlanner planner = new VolcanoPlannerX();
planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
planner.setTopDownOpt(true);
planner.setNoneConventionHasInfiniteCost(false);
config.getRules()
.forEach(
k -> {
RelRule.Config ruleConfig = null;
if (k.equals(ExtendIntersectRule.class.getSimpleName())) {
ruleConfig =
ExtendIntersectRule.Config.DEFAULT
.withMaxPatternSizeInGlogue(
config.getGlogueSize())
.withLabelConstraintsEnabled(
config.labelConstraintsEnabled());
} else if (k.equals(JoinDecompositionRule.class.getSimpleName())) {
ruleConfig =
JoinDecompositionRule.Config.DEFAULT
.withMinPatternSize(
config.getJoinMinPatternSize())
.withJoinQueueCapacity(
config.getJoinQueueCapacity())
.withJoinByEdgeEnabled(
config.isJoinByEdgeEnabled());
ForeignKeyMeta foreignKeyMeta =
config.getJoinByForeignKeyUri().isEmpty()
? null
: new ForeignKeyMeta(
config.getJoinByForeignKeyUri());
((JoinDecompositionRule.Config) ruleConfig)
.withForeignKeyMeta(foreignKeyMeta);
}
if (ruleConfig != null) {
planner.addRule(
ruleConfig
.withRelBuilderFactory(relBuilderFactory)
.toRule());
}
});
return planner;
}
// todo: re-implement heuristic rules in ir core match
return new HepPlanner(HepProgram.builder().build());
}

private RelOptPlanner createPhysicalPlanner() {
HepProgramBuilder hepBuilder = HepProgram.builder();
if (config.isOn()) {
List<RelRule.Config> ruleConfigs = Lists.newArrayList();
config.getRules()
.forEach(
k -> {
if (k.equals(ExpandGetVFusionRule.class.getSimpleName())) {
ruleConfigs.add(
ExpandGetVFusionRule.BasicExpandGetVFusionRule.Config
.DEFAULT);
ruleConfigs.add(
ExpandGetVFusionRule.PathBaseExpandGetVFusionRule.Config
.DEFAULT);
}
});
ruleConfigs.forEach(
k -> {
hepBuilder.addRuleInstance(
k.withRelBuilderFactory(relBuilderFactory).toRule());
});
}
return new GraphHepPlanner(hepBuilder.build());
}

private void clear() {
List<RelOptRule> logicalRBORules = this.relPlanner.getRules();
this.relPlanner.clear();
for (RelOptRule rule : logicalRBORules) {
this.relPlanner.addRule(rule);
}
List<RelOptRule> logicalCBORules = this.matchPlanner.getRules();
this.matchPlanner.clear();
for (RelOptRule rule : logicalCBORules) {
this.matchPlanner.addRule(rule);
}
List<RelOptRule> physicalRBORules = this.physicalPlanner.getRules();
this.physicalPlanner.clear();
for (RelOptRule rule : physicalRBORules) {
this.physicalPlanner.addRule(rule);
}
}
}
Loading

0 comments on commit 03fe273

Please sign in to comment.