Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(interactive): Distinguish 'label equals' or 'unique key equals' from Filter Conditions #3285

Merged
merged 5 commits into from
Oct 19, 2023
Merged

fix(interactive): Distinguish 'label equals' or 'unique key equals' from Filter Conditions #3285

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0d24045
[GIE Compiler] classify filters into label equals or unique key equal…
shirly121 Oct 12, 2023
230070b
[GIE Compiler] format java codes
shirly121 Oct 12, 2023
4318218
Merge branch 'main' into ir_classify_filters
longbinlai Oct 19, 2023
5a21901
Merge branch 'main' into ir_classify_filters
longbinlai Oct 19, 2023
7f0005e
[GIE Compiler] update expected results of unit test
shirly121 Oct 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion ...compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,15 @@
import org.apache.calcite.plan.GraphOptCluster;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.hint.RelHint;
import org.apache.calcite.rex.RexNode;
import org.checkerframework.checker.nullness.qual.Nullable;

import java.util.List;
import java.util.Objects;

public class GraphLogicalSource extends AbstractBindableTableScan {
private final GraphOpt.Source opt;
private @Nullable RexNode uniqueKeyFilters;

protected GraphLogicalSource(
GraphOptCluster cluster,
Expand All @@ -54,6 +57,16 @@ public GraphOpt.Source getOpt() {

@Override
public RelWriter explainTerms(RelWriter pw) {
return super.explainTerms(pw).item("opt", getOpt());
return super.explainTerms(pw)
.item("opt", getOpt())
.itemIf("uniqueKeyFilters", uniqueKeyFilters, uniqueKeyFilters != null);
}

public void setUniqueKeyFilters(RexNode uniqueKeyFilters) {
this.uniqueKeyFilters = Objects.requireNonNull(uniqueKeyFilters);
}

public @Nullable RexNode getUniqueKeyFilters() {
return uniqueKeyFilters;
}
}
62 changes: 0 additions & 62 deletions ...ine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java
View file
Open in desktop

This file was deleted.

3 changes: 2 additions & 1 deletion ...mpiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public FfiPhysicalBuilder(
Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan, PlanPointer planPointer) {
super(
logicalPlan,
new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId())));
new GraphRelShuttleWrapper(
new RelToFfiConverter(irMeta.getSchema().isColumnId(), graphConfig)));
this.graphConfig = graphConfig;
this.irMeta = irMeta;
this.planPointer = Objects.requireNonNull(planPointer);
Expand Down
144 changes: 96 additions & 48 deletions ...ompiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.alibaba.graphscope.common.ir.runtime.ffi;

import com.alibaba.graphscope.common.config.Configs;
import com.alibaba.graphscope.common.intermediate.ArgUtils;
import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate;
import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject;
Expand All @@ -31,8 +32,10 @@
import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter;
import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode;
import com.alibaba.graphscope.common.ir.tools.AliasInference;
import com.alibaba.graphscope.common.ir.tools.GraphPlanner;
import com.alibaba.graphscope.common.ir.tools.config.GraphOpt;
import com.alibaba.graphscope.common.ir.type.GraphLabelType;
import com.alibaba.graphscope.common.ir.type.GraphNameOrId;
import com.alibaba.graphscope.common.ir.type.GraphProperty;
import com.alibaba.graphscope.common.ir.type.GraphSchemaType;
import com.alibaba.graphscope.common.jna.IrCoreLibrary;
Expand All @@ -49,17 +52,13 @@
import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexVariable;
import org.apache.calcite.rex.*;
import org.apache.calcite.sql.SqlKind;
import org.apache.commons.lang3.ObjectUtils;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
Expand All @@ -72,9 +71,11 @@ public class RelToFfiConverter implements GraphRelShuttle {
private static final Logger logger = LoggerFactory.getLogger(RelToFfiConverter.class);
private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE;
private final boolean isColumnId;
private final RexBuilder rexBuilder;

public RelToFfiConverter(boolean isColumnId) {
public RelToFfiConverter(boolean isColumnId, Configs configs) {
this.isColumnId = isColumnId;
this.rexBuilder = GraphPlanner.rexBuilderFactory.apply(configs);
}

@Override
Expand Down Expand Up @@ -205,7 +206,9 @@ public RelNode visit(GraphLogicalMultiMatch match) {
@Override
public RelNode visit(LogicalFilter logicalFilter) {
OuterExpression.Expression exprProto =
logicalFilter.getCondition().accept(new RexToProtoConverter(true, isColumnId));
logicalFilter
.getCondition()
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder));
Pointer ptrFilter = LIB.initSelectOperator();
checkFfiResult(
LIB.setSelectPredicatePb(
Expand All @@ -219,7 +222,9 @@ public PhysicalNode visit(GraphLogicalProject project) {
List<RelDataTypeField> fields = project.getRowType().getFieldList();
for (int i = 0; i < project.getProjects().size(); ++i) {
OuterExpression.Expression expression =
project.getProjects().get(i).accept(new RexToProtoConverter(true, isColumnId));
project.getProjects()
.get(i)
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder));
int aliasId = fields.get(i).getIndex();
FfiAlias.ByValue ffiAlias =
(aliasId == AliasInference.DEFAULT_ID)
Expand Down Expand Up @@ -260,7 +265,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) {
RexGraphVariable.class,
var.getClass());
OuterExpression.Expression expr =
var.accept(new RexToProtoConverter(true, isColumnId));
var.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder));
int aliasId;
if (i >= fields.size()
|| (aliasId = fields.get(i).getIndex()) == AliasInference.DEFAULT_ID) {
Expand All @@ -283,7 +288,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) {
field.getName(),
field.getType());
OuterExpression.Variable exprVar =
rexVar.accept(new RexToProtoConverter(true, isColumnId))
rexVar.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder))
.getOperators(0)
.getVar();
checkFfiResult(
Expand All @@ -305,7 +310,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) {
OuterExpression.Variable var =
groupKeys
.get(i)
.accept(new RexToProtoConverter(true, isColumnId))
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder))
.getOperators(0)
.getVar();
int aliasId = fields.get(i).getIndex();
Expand Down Expand Up @@ -339,7 +344,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) {
operands.get(0).getClass());
OuterExpression.Variable var =
operands.get(0)
.accept(new RexToProtoConverter(true, isColumnId))
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder))
.getOperators(0)
.getVar();
checkFfiResult(
Expand Down Expand Up @@ -369,7 +374,7 @@ public PhysicalNode visit(GraphLogicalSort sort) {
for (int i = 0; i < collations.size(); ++i) {
RexGraphVariable expr = ((GraphFieldCollation) collations.get(i)).getVariable();
OuterExpression.Variable var =
expr.accept(new RexToProtoConverter(true, isColumnId))
expr.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder))
.getOperators(0)
.getVar();
checkFfiResult(
Expand Down Expand Up @@ -405,13 +410,13 @@ public PhysicalNode visit(LogicalJoin join) {
OuterExpression.Variable leftVar =
leftRightVars
.get(0)
.accept(new RexToProtoConverter(true, isColumnId))
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder))
.getOperators(0)
.getVar();
OuterExpression.Variable rightVar =
leftRightVars
.get(1)
.accept(new RexToProtoConverter(true, isColumnId))
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder))
.getOperators(0)
.getVar();
checkFfiResult(
Expand Down Expand Up @@ -451,7 +456,10 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) {
});
if (ObjectUtils.isNotEmpty(tableScan.getFilters())) {
OuterExpression.Expression expression =
tableScan.getFilters().get(0).accept(new RexToProtoConverter(true, isColumnId));
tableScan
.getFilters()
.get(0)
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder));
checkFfiResult(
LIB.setParamsPredicatePb(
params, new FfiPbPointer.ByValue(expression.toByteArray())));
Expand All @@ -460,46 +468,85 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) {
}

private @Nullable Pointer ffiIndexPredicates(GraphLogicalSource source) {
ImmutableList<RexNode> filters = source.getFilters();
if (ObjectUtils.isEmpty(filters)) return null;
// decomposed by OR
List<RexNode> disJunctions = RelOptUtil.disjunctions(filters.get(0));
List<RexNode> literals = new ArrayList<>();
for (RexNode rexNode : disJunctions) {
if (!isIdEqualsLiteral(rexNode, literals)) {
return null;
}
}
RexNode uniqueKeyFilters = source.getUniqueKeyFilters();
if (uniqueKeyFilters == null) return null;
// 'within' operator in index predicate is unsupported in ir core, here just expand it to
// 'or'
// i.e. '~id within [1, 2]' -> '~id == 1 or ~id == 2'
RexNode expandSearch = RexUtil.expandSearch(this.rexBuilder, null, uniqueKeyFilters);
List<RexNode> disjunctions = RelOptUtil.disjunctions(expandSearch);
Pointer ptrIndex = LIB.initIndexPredicate();
for (RexNode literal : literals) {
FfiProperty.ByValue property = new FfiProperty.ByValue();
property.opt = FfiPropertyOpt.Id;
checkFfiResult(
LIB.orEquivPredicate(ptrIndex, property, Utils.ffiConst((RexLiteral) literal)));
for (RexNode disjunction : disjunctions) {
if (disjunction instanceof RexCall) {
RexCall rexCall = (RexCall) disjunction;
switch (rexCall.getOperator().getKind()) {
case EQUALS:
RexNode left = rexCall.getOperands().get(0);
RexNode right = rexCall.getOperands().get(1);
if (left instanceof RexGraphVariable
&& (right instanceof RexLiteral
|| right instanceof RexDynamicParam)) {
LIB.orEquivPredicate(
ptrIndex,
getFfiProperty(((RexGraphVariable) left).getProperty()),
getFfiConst(right));
break;
} else if (right instanceof RexGraphVariable
&& (left instanceof RexLiteral
|| left instanceof RexDynamicParam)) {
LIB.orEquivPredicate(
ptrIndex,
getFfiProperty(((RexGraphVariable) right).getProperty()),
getFfiConst(left));
break;
}
default:
throw new IllegalArgumentException(
"can not convert unique key filter pattern="
+ rexCall
+ " to ir core index predicate");
}
} else {
throw new IllegalArgumentException(
"invalid unique key filter pattern=" + disjunction);
}
}
// remove index predicates from filter conditions
source.setFilters(ImmutableList.of());
return ptrIndex;
}

// i.e. a.~id == 10
private boolean isIdEqualsLiteral(RexNode rexNode, List<RexNode> literal) {
if (rexNode.getKind() != SqlKind.EQUALS) return false;
List<RexNode> operands = ((RexCall) rexNode).getOperands();
return isGlobalId(operands.get(0)) && isLiteral(operands.get(1), literal)
|| isGlobalId(operands.get(1)) && isLiteral(operands.get(0), literal);
private FfiProperty.ByValue getFfiProperty(GraphProperty property) {
Preconditions.checkArgument(property != null, "unique key should not be null");
FfiProperty.ByValue ffiProperty = new FfiProperty.ByValue();
switch (property.getOpt()) {
case ID:
ffiProperty.opt = FfiPropertyOpt.Id;
break;
case KEY:
ffiProperty.opt = FfiPropertyOpt.Key;
ffiProperty.key = getFfiNameOrId(property.getKey());
break;
default:
throw new IllegalArgumentException(
"can not convert property=" + property + " to ffi property");
}
return ffiProperty;
}

// i.e. a.~id
private boolean isGlobalId(RexNode rexNode) {
return rexNode instanceof RexGraphVariable
&& ((RexGraphVariable) rexNode).getProperty().getOpt() == GraphProperty.Opt.ID;
private FfiNameOrId.ByValue getFfiNameOrId(GraphNameOrId nameOrId) {
switch (nameOrId.getOpt()) {
case NAME:
return ArgUtils.asNameOrId(nameOrId.getName());
case ID:
default:
return ArgUtils.asNameOrId(nameOrId.getId());
}
}

private boolean isLiteral(RexNode rexNode, List<RexNode> literal) {
boolean isLiteral = rexNode.getKind() == SqlKind.LITERAL;
if (isLiteral) literal.add(rexNode);
return isLiteral;
private FfiConst.ByValue getFfiConst(RexNode rexNode) {
if (rexNode instanceof RexLiteral) {
return Utils.ffiConst((RexLiteral) rexNode);
}
throw new IllegalArgumentException("cannot convert rexNode=" + rexNode + " to ffi const");
}

private List<Integer> range(RexNode offset, RexNode fetch) {
Expand Down Expand Up @@ -592,7 +639,8 @@ private void addFilterToFfiBinder(Pointer ptrSentence, AbstractBindableTableScan
List<RexNode> filters = tableScan.getFilters();
if (ObjectUtils.isNotEmpty(filters)) {
OuterExpression.Expression exprProto =
filters.get(0).accept(new RexToProtoConverter(true, isColumnId));
filters.get(0)
.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder));
Pointer ptrFilter = LIB.initSelectOperator();
checkFfiResult(
LIB.setSelectPredicatePb(
Expand Down
Loading
Loading