Skip to content

Commit

Permalink
[GIE Compiler] convert index predicate from RexNode expression to ir …
Browse files Browse the repository at this point in the history 
…core
  • Loading branch information
@shirly121
shirly121 committed Oct 11, 2023
1 parent 87ee2ea commit d9b37f5
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 65 deletions.
9 changes: 0 additions & 9 deletions ...compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
public class GraphLogicalSource extends AbstractBindableTableScan {
private final GraphOpt.Source opt;
private @Nullable RexNode uniqueKeyFilters;
private @Nullable RexNode primaryKeyFilters;

protected GraphLogicalSource(
GraphOptCluster cluster,
Expand Down Expand Up @@ -67,15 +66,7 @@ public void setUniqueKeyFilters(RexNode uniqueKeyFilters) {
this.uniqueKeyFilters = Objects.requireNonNull(uniqueKeyFilters);
}

public void setPrimaryKeyFilters(RexNode primaryKeyFilters) {
this.primaryKeyFilters = Objects.requireNonNull(primaryKeyFilters);
}

public @Nullable RexNode getUniqueKeyFilters() {
return uniqueKeyFilters;
}

public @Nullable RexNode getPrimaryKeyFilters() {
return primaryKeyFilters;
}
}
3 changes: 2 additions & 1 deletion ...mpiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public FfiPhysicalBuilder(
Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan, PlanPointer planPointer) {
super(
logicalPlan,
new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId())));
new GraphRelShuttleWrapper(
new RelToFfiConverter(irMeta.getSchema().isColumnId(), graphConfig)));
this.graphConfig = graphConfig;
this.irMeta = irMeta;
this.planPointer = Objects.requireNonNull(planPointer);
Expand Down
114 changes: 77 additions & 37 deletions ...ompiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.alibaba.graphscope.common.ir.runtime.ffi;

import com.alibaba.graphscope.common.config.Configs;
import com.alibaba.graphscope.common.intermediate.ArgUtils;
import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate;
import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject;
Expand All @@ -31,8 +32,10 @@
import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter;
import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode;
import com.alibaba.graphscope.common.ir.tools.AliasInference;
import com.alibaba.graphscope.common.ir.tools.GraphPlanner;
import com.alibaba.graphscope.common.ir.tools.config.GraphOpt;
import com.alibaba.graphscope.common.ir.type.GraphLabelType;
import com.alibaba.graphscope.common.ir.type.GraphNameOrId;
import com.alibaba.graphscope.common.ir.type.GraphProperty;
import com.alibaba.graphscope.common.ir.type.GraphSchemaType;
import com.alibaba.graphscope.common.jna.IrCoreLibrary;
Expand All @@ -49,17 +52,13 @@
import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexVariable;
import org.apache.calcite.rex.*;
import org.apache.calcite.sql.SqlKind;
import org.apache.commons.lang3.ObjectUtils;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
Expand All @@ -72,9 +71,11 @@ public class RelToFfiConverter implements GraphRelShuttle {
private static final Logger logger = LoggerFactory.getLogger(RelToFfiConverter.class);
private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE;
private final boolean isColumnId;
private final RexBuilder rexBuilder;

public RelToFfiConverter(boolean isColumnId) {
public RelToFfiConverter(boolean isColumnId, Configs configs) {
this.isColumnId = isColumnId;
this.rexBuilder = GraphPlanner.rexBuilderFactory.apply(configs);
}

@Override
Expand Down Expand Up @@ -460,46 +461,85 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) {
}

private @Nullable Pointer ffiIndexPredicates(GraphLogicalSource source) {
ImmutableList<RexNode> filters = source.getFilters();
if (ObjectUtils.isEmpty(filters)) return null;
// decomposed by OR
List<RexNode> disJunctions = RelOptUtil.disjunctions(filters.get(0));
List<RexNode> literals = new ArrayList<>();
for (RexNode rexNode : disJunctions) {
if (!isIdEqualsLiteral(rexNode, literals)) {
return null;
}
}
RexNode uniqueKeyFilters = source.getUniqueKeyFilters();
if (uniqueKeyFilters == null) return null;
// 'within' operator in index predicate is unsupported in ir core, here just expand it to
// 'or'
// i.e. '~id within [1, 2]' -> '~id == 1 or ~id == 2'
RexNode expandSearch = RexUtil.expandSearch(this.rexBuilder, null, uniqueKeyFilters);
List<RexNode> disjunctions = RelOptUtil.disjunctions(expandSearch);
Pointer ptrIndex = LIB.initIndexPredicate();
for (RexNode literal : literals) {
FfiProperty.ByValue property = new FfiProperty.ByValue();
property.opt = FfiPropertyOpt.Id;
checkFfiResult(
LIB.orEquivPredicate(ptrIndex, property, Utils.ffiConst((RexLiteral) literal)));
for (RexNode disjunction : disjunctions) {
if (disjunction instanceof RexCall) {
RexCall rexCall = (RexCall) disjunction;
switch (rexCall.getOperator().getKind()) {
case EQUALS:
RexNode left = rexCall.getOperands().get(0);
RexNode right = rexCall.getOperands().get(1);
if (left instanceof RexGraphVariable
&& (right instanceof RexLiteral
|| right instanceof RexDynamicParam)) {
LIB.orEquivPredicate(
ptrIndex,
getFfiProperty(((RexGraphVariable) left).getProperty()),
getFfiConst(right));
break;
} else if (right instanceof RexGraphVariable
&& (left instanceof RexLiteral
|| left instanceof RexDynamicParam)) {
LIB.orEquivPredicate(
ptrIndex,
getFfiProperty(((RexGraphVariable) right).getProperty()),
getFfiConst(left));
break;
}
default:
throw new IllegalArgumentException(
"can not convert unique key filter pattern="
+ rexCall
+ " to ir core index predicate");
}
} else {
throw new IllegalArgumentException(
"invalid unique key filter pattern=" + disjunction);
}
}
// remove index predicates from filter conditions
source.setFilters(ImmutableList.of());
return ptrIndex;
}

// i.e. a.~id == 10
private boolean isIdEqualsLiteral(RexNode rexNode, List<RexNode> literal) {
if (rexNode.getKind() != SqlKind.EQUALS) return false;
List<RexNode> operands = ((RexCall) rexNode).getOperands();
return isGlobalId(operands.get(0)) && isLiteral(operands.get(1), literal)
|| isGlobalId(operands.get(1)) && isLiteral(operands.get(0), literal);
private FfiProperty.ByValue getFfiProperty(GraphProperty property) {
Preconditions.checkArgument(property != null, "unique key should not be null");
FfiProperty.ByValue ffiProperty = new FfiProperty.ByValue();
switch (property.getOpt()) {
case ID:
ffiProperty.opt = FfiPropertyOpt.Id;
break;
case KEY:
ffiProperty.opt = FfiPropertyOpt.Key;
ffiProperty.key = getFfiNameOrId(property.getKey());
break;
default:
throw new IllegalArgumentException(
"can not convert property=" + property + " to ffi property");
}
return ffiProperty;
}

// i.e. a.~id
private boolean isGlobalId(RexNode rexNode) {
return rexNode instanceof RexGraphVariable
&& ((RexGraphVariable) rexNode).getProperty().getOpt() == GraphProperty.Opt.ID;
private FfiNameOrId.ByValue getFfiNameOrId(GraphNameOrId nameOrId) {
switch (nameOrId.getOpt()) {
case NAME:
return ArgUtils.asNameOrId(nameOrId.getName());
case ID:
default:
return ArgUtils.asNameOrId(nameOrId.getId());
}
}

private boolean isLiteral(RexNode rexNode, List<RexNode> literal) {
boolean isLiteral = rexNode.getKind() == SqlKind.LITERAL;
if (isLiteral) literal.add(rexNode);
return isLiteral;
private FfiConst.ByValue getFfiConst(RexNode rexNode) {
if (rexNode instanceof RexLiteral) {
return Utils.ffiConst((RexLiteral) rexNode);
}
throw new IllegalArgumentException("cannot convert rexNode=" + rexNode + " to ffi const");
}

private List<Integer> range(RexNode offset, RexNode fetch) {
Expand Down
51 changes: 35 additions & 16 deletions ...ve_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -769,9 +769,11 @@ public Void visitInputRef(RexInputRef inputRef) {
}
};
if (tableScan instanceof GraphLogicalSource) {
GraphLogicalSource source = (GraphLogicalSource) tableScan;
if (source.getUniqueKeyFilters() != null) {
builder.filter(source.getUniqueKeyFilters());
RexNode originalUniqueKeyFilters =
((GraphLogicalSource) tableScan).getUniqueKeyFilters();
if (originalUniqueKeyFilters != null) {
originalUniqueKeyFilters.accept(propertyChecker);
builder.filter(originalUniqueKeyFilters);
}
}
ImmutableList originalFilters = tableScan.getFilters();
Expand All @@ -788,12 +790,11 @@ public Void visitInputRef(RexInputRef inputRef) {
}
if (tableScan instanceof GraphLogicalSource && !uniqueKeyFilters.isEmpty()) {
GraphLogicalSource source = (GraphLogicalSource) tableScan;
if (source.getUniqueKeyFilters() != null) {
uniqueKeyFilters.add(0, source.getUniqueKeyFilters());
}
// todo: check if unique key filters have common keys, otherwise throw errors
Preconditions.checkArgument(
source.getUniqueKeyFilters() == null,
"can not add unique key filters if original is not empty");
source.setUniqueKeyFilters(
RexUtil.composeConjunction(this.getRexBuilder(), uniqueKeyFilters));
RexUtil.composeDisjunction(this.getRexBuilder(), uniqueKeyFilters));
}
if (!extraFilters.isEmpty()) {
ImmutableList originalFilters = tableScan.getFilters();
Expand All @@ -813,7 +814,7 @@ private void classifyFilters(
AbstractBindableTableScan tableScan,
RexNode condition,
List<Comparable> labelValues,
List<RexNode> uniqueKeyFilters,
List<RexNode> uniqueKeyFilters, // unique key filters int the list are composed by 'OR'
List<RexNode> filters) {
List<RexNode> conjunctions = RelOptUtil.conjunctions(condition);
List<RexNode> filtersToRemove = Lists.newArrayList();
Expand All @@ -836,13 +837,26 @@ private void classifyFilters(
getValuesAsList(((RexLiteral) left).getValueAs(Comparable.class)));
break;
}
if (tableScan instanceof GraphLogicalSource) {
if (isUniqueKey(left) && right instanceof RexLiteral) {
filtersToRemove.add(conjunction);
uniqueKeyFilters.add(conjunction);
} else if (left instanceof RexLiteral && isUniqueKey(right)) {
filtersToRemove.add(conjunction);
uniqueKeyFilters.add(conjunction);
}
}
}
if (tableScan instanceof GraphLogicalSource
&& ((GraphLogicalSource) tableScan).getUniqueKeyFilters() == null) {
// try to extract unique key filters from the original condition
List<RexNode> disjunctions = RelOptUtil.disjunctions(condition);
for (RexNode disjunction : disjunctions) {
if (disjunction instanceof RexCall) {
RexCall rexCall = (RexCall) disjunction;
if (rexCall.getOperator().getKind() == SqlKind.EQUALS
|| rexCall.getOperator().getKind() == SqlKind.SEARCH) {
RexNode left = rexCall.getOperands().get(0);
RexNode right = rexCall.getOperands().get(1);
if (isUniqueKey(left) && isLiteralOrDynamicParams(right)) {
filtersToRemove.add(disjunction);
uniqueKeyFilters.add(disjunction);
} else if (isLiteralOrDynamicParams(left) && isUniqueKey(right)) {
filtersToRemove.add(disjunction);
uniqueKeyFilters.add(disjunction);
}
}
}
Expand All @@ -855,6 +869,7 @@ private void classifyFilters(
}

private boolean isUniqueKey(RexNode rexNode) {
// todo: support primary keys
if (rexNode instanceof RexGraphVariable) {
RexGraphVariable variable = (RexGraphVariable) rexNode;
return variable.getProperty() != null
Expand All @@ -863,6 +878,10 @@ private boolean isUniqueKey(RexNode rexNode) {
return false;
}

private boolean isLiteralOrDynamicParams(RexNode node) {
return node instanceof RexLiteral || node instanceof RexDynamicParam;
}

private List<Comparable> getValuesAsList(Comparable value) {
ImmutableList.Builder labelBuilder = ImmutableList.builder();
if (value instanceof NlsString) {
Expand Down
7 changes: 5 additions & 2 deletions ...ve_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

/**
* A unified structure to build {@link PlannerInstance} which can further build logical and physical plan from an antlr tree
Expand All @@ -70,18 +71,20 @@ public class GraphPlanner {
private final Configs graphConfig;
private final PlannerConfig plannerConfig;
private final RelOptPlanner optPlanner;
private final RexBuilder rexBuilder;
private final AtomicLong idGenerator;
private final RexBuilder rexBuilder;
public static final RelBuilderFactory relBuilderFactory =
(RelOptCluster cluster, @Nullable RelOptSchema schema) ->
GraphBuilder.create(null, (GraphOptCluster) cluster, schema);
public static final Function<Configs, RexBuilder> rexBuilderFactory =
(Configs configs) -> new GraphRexBuilder(new GraphTypeFactoryImpl(configs));

public GraphPlanner(Configs graphConfig) {
this.graphConfig = graphConfig;
this.plannerConfig = PlannerConfig.create(this.graphConfig);
logger.debug("planner config: " + this.plannerConfig);
this.optPlanner = createRelOptPlanner(this.plannerConfig);
this.rexBuilder = new GraphRexBuilder(new GraphTypeFactoryImpl(graphConfig));
this.rexBuilder = rexBuilderFactory.apply(graphConfig);
this.idGenerator = new AtomicLong(FrontendConfig.FRONTEND_SERVER_ID.get(graphConfig));
}

Expand Down

0 comments on commit d9b37f5

Please sign in to comment.