Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(interactive): Fix Concurrent Bugs in Graph Optimizer #4269

Merged
merged 3 commits into from
Sep 29, 2024
Merged

fix(interactive): Fix Concurrent Bugs in Graph Optimizer #4269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
750e2d3
[GIE Compiler] fix bugs of CBO for concurrent queries
shirly121 Sep 4, 2024
14999a0
fix JoinDecompositionRule in path expand
shirly121 Sep 27, 2024
fe55805
fix ci tests
shirly121 Sep 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions ...ive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/PlannerConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ public class PlannerConfig {
// in Neo4j
public static final Config<Boolean> JOIN_BY_EDGE_ENABLED =
Config.boolConfig("graph.planner.join.by.edge.enabled", false);
public static final Config<Integer> GRAPH_PLANNER_GROUP_SIZE =
Config.intConfig("graph.planner.group.size", 8);
public static final Config<Integer> GRAPH_PLANNER_GROUP_CLEAR_INTERVAL_MINUTES =
Config.intConfig("graph.planner.group.clear.interval.minutes", 30);

private final Configs configs;
private final List<String> rules;
Expand Down Expand Up @@ -96,6 +100,14 @@ public String getJoinByForeignKeyUri() {
return GraphConfig.GRAPH_FOREIGN_KEY_URI.get(configs);
}

public int getPlannerGroupSize() {
return GRAPH_PLANNER_GROUP_SIZE.get(configs);
}

public int getPlannerGroupClearIntervalMinutes() {
return GRAPH_PLANNER_GROUP_CLEAR_INTERVAL_MINUTES.get(configs);
}

@Override
public String toString() {
return "PlannerConfig{"
Expand Down
191 changes: 27 additions & 164 deletions ...ne/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,6 @@
import com.alibaba.graphscope.common.ir.meta.IrMeta;
import com.alibaba.graphscope.common.ir.meta.glogue.calcite.GraphRelMetadataQuery;
import com.alibaba.graphscope.common.ir.meta.glogue.calcite.handler.GraphMetadataHandlerProvider;
import com.alibaba.graphscope.common.ir.meta.schema.foreign.ForeignKeyMeta;
import com.alibaba.graphscope.common.ir.planner.rules.*;
import com.alibaba.graphscope.common.ir.planner.volcano.VolcanoPlannerX;
import com.alibaba.graphscope.common.ir.rel.GraphShuttle;
import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource;
import com.alibaba.graphscope.common.ir.rel.graph.match.AbstractLogicalMatch;
Expand All @@ -36,19 +33,14 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;

import org.apache.calcite.plan.*;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgram;
import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.plan.GraphOptCluster;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rel.rules.FilterJoinRule;
import org.apache.calcite.tools.RelBuilderFactory;
import org.checkerframework.checker.nullness.qual.Nullable;

Expand All @@ -59,75 +51,59 @@
* Optimize graph relational tree which consists of match and other relational operators
*/
public class GraphRelOptimizer {
private final Configs graphConfig;
private final PlannerConfig config;
private final RelOptPlanner relPlanner;
private final RelOptPlanner matchPlanner;
private final RelOptPlanner physicalPlanner;
private final RelBuilderFactory relBuilderFactory;
private final GlogueHolder glogueHolder;
private final PlannerGroupManager plannerGroupManager;

public GraphRelOptimizer(Configs graphConfig, Class<? extends PlannerGroupManager> instance) {
try {
this.config = new PlannerConfig(graphConfig);
this.relBuilderFactory = new GraphBuilderFactory(graphConfig);
this.glogueHolder = new GlogueHolder(graphConfig);
this.plannerGroupManager =
instance.getDeclaredConstructor(PlannerConfig.class, RelBuilderFactory.class)
.newInstance(this.config, this.relBuilderFactory);
} catch (Exception e) {
throw new RuntimeException(e);
}
}

public GraphRelOptimizer(Configs graphConfig) {
this.graphConfig = graphConfig;
this.config = new PlannerConfig(graphConfig);
this.relBuilderFactory = new GraphBuilderFactory(graphConfig);
this.relPlanner = createRelPlanner();
this.matchPlanner = createMatchPlanner();
this.physicalPlanner = createPhysicalPlanner();
this.glogueHolder = new GlogueHolder(graphConfig);
this(graphConfig, PlannerGroupManager.Dynamic.class);
}

public GlogueHolder getGlogueHolder() {
return glogueHolder;
}

public RelOptPlanner getMatchPlanner() {
return matchPlanner;
}

public RelOptPlanner getPhysicalPlanner() {
return physicalPlanner;
PlannerGroup currentGroup = this.plannerGroupManager.getCurrentGroup();
return currentGroup.getMatchPlanner();
}

public RelOptPlanner getRelPlanner() {
return relPlanner;
public RelNode optimize(RelNode before, GraphIOProcessor ioProcessor) {
PlannerGroup currentGroup = this.plannerGroupManager.getCurrentGroup();
return currentGroup.optimize(before, ioProcessor);
}

public @Nullable RelMetadataQuery createMetaDataQuery(IrMeta irMeta) {
if (config.isOn() && config.getOpt() == PlannerConfig.Opt.CBO) {
GlogueQuery gq = this.glogueHolder.getGlogue();
Preconditions.checkArgument(gq != null, "glogue is not ready");
return new GraphRelMetadataQuery(
new GraphMetadataHandlerProvider(this.matchPlanner, gq, this.config));
new GraphMetadataHandlerProvider(getMatchPlanner(), gq, this.config));
}
return null;
}

public RelNode optimize(RelNode before, GraphIOProcessor ioProcessor) {
if (config.isOn()) {
// apply rules of 'FilterPushDown' before the match optimization
relPlanner.setRoot(before);
RelNode relOptimized = relPlanner.findBestExp();
if (config.getOpt() == PlannerConfig.Opt.CBO) {
relOptimized = relOptimized.accept(new MatchOptimizer(ioProcessor));
}
// apply rules of 'FieldTrim' after the match optimization
if (config.getRules().contains(FieldTrimRule.class.getSimpleName())) {
relOptimized = FieldTrimRule.trim(ioProcessor.getBuilder(), relOptimized);
}
physicalPlanner.setRoot(relOptimized);
RelNode physicalOptimized = physicalPlanner.findBestExp();
clear();
return physicalOptimized;
}
return before;
}

private class MatchOptimizer extends GraphShuttle {
public static class MatchOptimizer extends GraphShuttle {
private final GraphIOProcessor ioProcessor;
private final RelOptPlanner matchPlanner;

public MatchOptimizer(GraphIOProcessor ioProcessor) {
public MatchOptimizer(GraphIOProcessor ioProcessor, RelOptPlanner matchPlanner) {
this.ioProcessor = ioProcessor;
this.matchPlanner = matchPlanner;
}

@Override
Expand Down Expand Up @@ -219,117 +195,4 @@ public void visit(RelNode node, int ordinal, @Nullable RelNode parent) {
return decomposable.get();
}
}

private RelOptPlanner createRelPlanner() {
HepProgramBuilder hepBuilder = HepProgram.builder();
if (config.isOn()) {
List<RelRule.Config> ruleConfigs = Lists.newArrayList();
config.getRules()
.forEach(
k -> {
if (k.equals(
FilterJoinRule.FilterIntoJoinRule.class.getSimpleName())) {
ruleConfigs.add(CoreRules.FILTER_INTO_JOIN.config);
} else if (k.equals(FilterMatchRule.class.getSimpleName())) {
ruleConfigs.add(FilterMatchRule.Config.DEFAULT);
}
});
ruleConfigs.forEach(
k -> {
hepBuilder.addRuleInstance(
k.withRelBuilderFactory(relBuilderFactory).toRule());
});
}
return new HepPlanner(hepBuilder.build());
}

private RelOptPlanner createMatchPlanner() {
if (config.isOn() && config.getOpt() == PlannerConfig.Opt.CBO) {
VolcanoPlanner planner = new VolcanoPlannerX();
planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
planner.setTopDownOpt(true);
planner.setNoneConventionHasInfiniteCost(false);
config.getRules()
.forEach(
k -> {
RelRule.Config ruleConfig = null;
if (k.equals(ExtendIntersectRule.class.getSimpleName())) {
ruleConfig =
ExtendIntersectRule.Config.DEFAULT
.withMaxPatternSizeInGlogue(
config.getGlogueSize())
.withLabelConstraintsEnabled(
config.labelConstraintsEnabled());
} else if (k.equals(JoinDecompositionRule.class.getSimpleName())) {
ruleConfig =
JoinDecompositionRule.Config.DEFAULT
.withMinPatternSize(
config.getJoinMinPatternSize())
.withJoinQueueCapacity(
config.getJoinQueueCapacity())
.withJoinByEdgeEnabled(
config.isJoinByEdgeEnabled());
ForeignKeyMeta foreignKeyMeta =
config.getJoinByForeignKeyUri().isEmpty()
? null
: new ForeignKeyMeta(
config.getJoinByForeignKeyUri());
((JoinDecompositionRule.Config) ruleConfig)
.withForeignKeyMeta(foreignKeyMeta);
}
if (ruleConfig != null) {
planner.addRule(
ruleConfig
.withRelBuilderFactory(relBuilderFactory)
.toRule());
}
});
return planner;
}
// todo: re-implement heuristic rules in ir core match
return new HepPlanner(HepProgram.builder().build());
}

private RelOptPlanner createPhysicalPlanner() {
HepProgramBuilder hepBuilder = HepProgram.builder();
if (config.isOn()) {
List<RelRule.Config> ruleConfigs = Lists.newArrayList();
config.getRules()
.forEach(
k -> {
if (k.equals(ExpandGetVFusionRule.class.getSimpleName())) {
ruleConfigs.add(
ExpandGetVFusionRule.BasicExpandGetVFusionRule.Config
.DEFAULT);
ruleConfigs.add(
ExpandGetVFusionRule.PathBaseExpandGetVFusionRule.Config
.DEFAULT);
}
});
ruleConfigs.forEach(
k -> {
hepBuilder.addRuleInstance(
k.withRelBuilderFactory(relBuilderFactory).toRule());
});
}
return new GraphHepPlanner(hepBuilder.build());
}

private void clear() {
List<RelOptRule> logicalRBORules = this.relPlanner.getRules();
this.relPlanner.clear();
for (RelOptRule rule : logicalRBORules) {
this.relPlanner.addRule(rule);
}
List<RelOptRule> logicalCBORules = this.matchPlanner.getRules();
this.matchPlanner.clear();
for (RelOptRule rule : logicalCBORules) {
this.matchPlanner.addRule(rule);
}
List<RelOptRule> physicalRBORules = this.physicalPlanner.getRules();
this.physicalPlanner.clear();
for (RelOptRule rule : physicalRBORules) {
this.physicalPlanner.addRule(rule);
}
}
}
Loading
Loading