diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java index 5d5b3bbb1c94..2d6fce56a41c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java @@ -31,7 +31,6 @@ public class GraphLogicalSource extends AbstractBindableTableScan { private final GraphOpt.Source opt; private @Nullable RexNode uniqueKeyFilters; - private @Nullable RexNode primaryKeyFilters; protected GraphLogicalSource( GraphOptCluster cluster, @@ -67,15 +66,7 @@ public void setUniqueKeyFilters(RexNode uniqueKeyFilters) { this.uniqueKeyFilters = Objects.requireNonNull(uniqueKeyFilters); } - public void setPrimaryKeyFilters(RexNode primaryKeyFilters) { - this.primaryKeyFilters = Objects.requireNonNull(primaryKeyFilters); - } - public @Nullable RexNode getUniqueKeyFilters() { return uniqueKeyFilters; } - - public @Nullable RexNode getPrimaryKeyFilters() { - return primaryKeyFilters; - } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java index 9216217d96da..b05ce9b46651 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java @@ -68,7 +68,8 @@ public FfiPhysicalBuilder( Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan, PlanPointer planPointer) { super( logicalPlan, - new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId()))); + new GraphRelShuttleWrapper( + new RelToFfiConverter(irMeta.getSchema().isColumnId(), graphConfig))); this.graphConfig = graphConfig; this.irMeta = irMeta; this.planPointer = Objects.requireNonNull(planPointer); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java index f9efd30c80c9..fb044d9e4402 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.common.ir.runtime.ffi; +import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.intermediate.ArgUtils; import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate; import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject; @@ -31,8 +32,10 @@ import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter; import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode; import com.alibaba.graphscope.common.ir.tools.AliasInference; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.alibaba.graphscope.common.ir.type.GraphNameOrId; import com.alibaba.graphscope.common.ir.type.GraphProperty; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.common.jna.IrCoreLibrary; @@ -49,17 +52,13 @@ import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexVariable; +import org.apache.calcite.rex.*; import org.apache.calcite.sql.SqlKind; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -72,9 +71,11 @@ public class RelToFfiConverter implements GraphRelShuttle { private static final Logger logger = LoggerFactory.getLogger(RelToFfiConverter.class); private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE; private final boolean isColumnId; + private final RexBuilder rexBuilder; - public RelToFfiConverter(boolean isColumnId) { + public RelToFfiConverter(boolean isColumnId, Configs configs) { this.isColumnId = isColumnId; + this.rexBuilder = GraphPlanner.rexBuilderFactory.apply(configs); } @Override @@ -460,46 +461,85 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) { } private @Nullable Pointer ffiIndexPredicates(GraphLogicalSource source) { - ImmutableList filters = source.getFilters(); - if (ObjectUtils.isEmpty(filters)) return null; - // decomposed by OR - List disJunctions = RelOptUtil.disjunctions(filters.get(0)); - List literals = new ArrayList<>(); - for (RexNode rexNode : disJunctions) { - if (!isIdEqualsLiteral(rexNode, literals)) { - return null; - } - } + RexNode uniqueKeyFilters = source.getUniqueKeyFilters(); + if (uniqueKeyFilters == null) return null; + // 'within' operator in index predicate is unsupported in ir core, here just expand it to + // 'or' + // i.e. '~id within [1, 2]' -> '~id == 1 or ~id == 2' + RexNode expandSearch = RexUtil.expandSearch(this.rexBuilder, null, uniqueKeyFilters); + List disjunctions = RelOptUtil.disjunctions(expandSearch); Pointer ptrIndex = LIB.initIndexPredicate(); - for (RexNode literal : literals) { - FfiProperty.ByValue property = new FfiProperty.ByValue(); - property.opt = FfiPropertyOpt.Id; - checkFfiResult( - LIB.orEquivPredicate(ptrIndex, property, Utils.ffiConst((RexLiteral) literal))); + for (RexNode disjunction : disjunctions) { + if (disjunction instanceof RexCall) { + RexCall rexCall = (RexCall) disjunction; + switch (rexCall.getOperator().getKind()) { + case EQUALS: + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (left instanceof RexGraphVariable + && (right instanceof RexLiteral + || right instanceof RexDynamicParam)) { + LIB.orEquivPredicate( + ptrIndex, + getFfiProperty(((RexGraphVariable) left).getProperty()), + getFfiConst(right)); + break; + } else if (right instanceof RexGraphVariable + && (left instanceof RexLiteral + || left instanceof RexDynamicParam)) { + LIB.orEquivPredicate( + ptrIndex, + getFfiProperty(((RexGraphVariable) right).getProperty()), + getFfiConst(left)); + break; + } + default: + throw new IllegalArgumentException( + "can not convert unique key filter pattern=" + + rexCall + + " to ir core index predicate"); + } + } else { + throw new IllegalArgumentException( + "invalid unique key filter pattern=" + disjunction); + } } - // remove index predicates from filter conditions - source.setFilters(ImmutableList.of()); return ptrIndex; } - // i.e. a.~id == 10 - private boolean isIdEqualsLiteral(RexNode rexNode, List literal) { - if (rexNode.getKind() != SqlKind.EQUALS) return false; - List operands = ((RexCall) rexNode).getOperands(); - return isGlobalId(operands.get(0)) && isLiteral(operands.get(1), literal) - || isGlobalId(operands.get(1)) && isLiteral(operands.get(0), literal); + private FfiProperty.ByValue getFfiProperty(GraphProperty property) { + Preconditions.checkArgument(property != null, "unique key should not be null"); + FfiProperty.ByValue ffiProperty = new FfiProperty.ByValue(); + switch (property.getOpt()) { + case ID: + ffiProperty.opt = FfiPropertyOpt.Id; + break; + case KEY: + ffiProperty.opt = FfiPropertyOpt.Key; + ffiProperty.key = getFfiNameOrId(property.getKey()); + break; + default: + throw new IllegalArgumentException( + "can not convert property=" + property + " to ffi property"); + } + return ffiProperty; } - // i.e. a.~id - private boolean isGlobalId(RexNode rexNode) { - return rexNode instanceof RexGraphVariable - && ((RexGraphVariable) rexNode).getProperty().getOpt() == GraphProperty.Opt.ID; + private FfiNameOrId.ByValue getFfiNameOrId(GraphNameOrId nameOrId) { + switch (nameOrId.getOpt()) { + case NAME: + return ArgUtils.asNameOrId(nameOrId.getName()); + case ID: + default: + return ArgUtils.asNameOrId(nameOrId.getId()); + } } - private boolean isLiteral(RexNode rexNode, List literal) { - boolean isLiteral = rexNode.getKind() == SqlKind.LITERAL; - if (isLiteral) literal.add(rexNode); - return isLiteral; + private FfiConst.ByValue getFfiConst(RexNode rexNode) { + if (rexNode instanceof RexLiteral) { + return Utils.ffiConst((RexLiteral) rexNode); + } + throw new IllegalArgumentException("cannot convert rexNode=" + rexNode + " to ffi const"); } private List range(RexNode offset, RexNode fetch) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index be83a0cc7b2a..d917e58d0b59 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -769,9 +769,11 @@ public Void visitInputRef(RexInputRef inputRef) { } }; if (tableScan instanceof GraphLogicalSource) { - GraphLogicalSource source = (GraphLogicalSource) tableScan; - if (source.getUniqueKeyFilters() != null) { - builder.filter(source.getUniqueKeyFilters()); + RexNode originalUniqueKeyFilters = + ((GraphLogicalSource) tableScan).getUniqueKeyFilters(); + if (originalUniqueKeyFilters != null) { + originalUniqueKeyFilters.accept(propertyChecker); + builder.filter(originalUniqueKeyFilters); } } ImmutableList originalFilters = tableScan.getFilters(); @@ -788,12 +790,11 @@ public Void visitInputRef(RexInputRef inputRef) { } if (tableScan instanceof GraphLogicalSource && !uniqueKeyFilters.isEmpty()) { GraphLogicalSource source = (GraphLogicalSource) tableScan; - if (source.getUniqueKeyFilters() != null) { - uniqueKeyFilters.add(0, source.getUniqueKeyFilters()); - } - // todo: check if unique key filters have common keys, otherwise throw errors + Preconditions.checkArgument( + source.getUniqueKeyFilters() == null, + "can not add unique key filters if original is not empty"); source.setUniqueKeyFilters( - RexUtil.composeConjunction(this.getRexBuilder(), uniqueKeyFilters)); + RexUtil.composeDisjunction(this.getRexBuilder(), uniqueKeyFilters)); } if (!extraFilters.isEmpty()) { ImmutableList originalFilters = tableScan.getFilters(); @@ -813,7 +814,7 @@ private void classifyFilters( AbstractBindableTableScan tableScan, RexNode condition, List labelValues, - List uniqueKeyFilters, + List uniqueKeyFilters, // unique key filters int the list are composed by 'OR' List filters) { List conjunctions = RelOptUtil.conjunctions(condition); List filtersToRemove = Lists.newArrayList(); @@ -836,13 +837,26 @@ private void classifyFilters( getValuesAsList(((RexLiteral) left).getValueAs(Comparable.class))); break; } - if (tableScan instanceof GraphLogicalSource) { - if (isUniqueKey(left) && right instanceof RexLiteral) { - filtersToRemove.add(conjunction); - uniqueKeyFilters.add(conjunction); - } else if (left instanceof RexLiteral && isUniqueKey(right)) { - filtersToRemove.add(conjunction); - uniqueKeyFilters.add(conjunction); + } + } + } + if (tableScan instanceof GraphLogicalSource + && ((GraphLogicalSource) tableScan).getUniqueKeyFilters() == null) { + // try to extract unique key filters from the original condition + List disjunctions = RelOptUtil.disjunctions(condition); + for (RexNode disjunction : disjunctions) { + if (disjunction instanceof RexCall) { + RexCall rexCall = (RexCall) disjunction; + if (rexCall.getOperator().getKind() == SqlKind.EQUALS + || rexCall.getOperator().getKind() == SqlKind.SEARCH) { + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (isUniqueKey(left) && isLiteralOrDynamicParams(right)) { + filtersToRemove.add(disjunction); + uniqueKeyFilters.add(disjunction); + } else if (isLiteralOrDynamicParams(left) && isUniqueKey(right)) { + filtersToRemove.add(disjunction); + uniqueKeyFilters.add(disjunction); } } } @@ -855,6 +869,7 @@ private void classifyFilters( } private boolean isUniqueKey(RexNode rexNode) { + // todo: support primary keys if (rexNode instanceof RexGraphVariable) { RexGraphVariable variable = (RexGraphVariable) rexNode; return variable.getProperty() != null @@ -863,6 +878,10 @@ private boolean isUniqueKey(RexNode rexNode) { return false; } + private boolean isLiteralOrDynamicParams(RexNode node) { + return node instanceof RexLiteral || node instanceof RexDynamicParam; + } + private List getValuesAsList(Comparable value) { ImmutableList.Builder labelBuilder = ImmutableList.builder(); if (value instanceof NlsString) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 4a2ef40d8821..3cba4882d53a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -61,6 +61,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; /** * A unified structure to build {@link PlannerInstance} which can further build logical and physical plan from an antlr tree @@ -70,18 +71,20 @@ public class GraphPlanner { private final Configs graphConfig; private final PlannerConfig plannerConfig; private final RelOptPlanner optPlanner; - private final RexBuilder rexBuilder; private final AtomicLong idGenerator; + private final RexBuilder rexBuilder; public static final RelBuilderFactory relBuilderFactory = (RelOptCluster cluster, @Nullable RelOptSchema schema) -> GraphBuilder.create(null, (GraphOptCluster) cluster, schema); + public static final Function rexBuilderFactory = + (Configs configs) -> new GraphRexBuilder(new GraphTypeFactoryImpl(configs)); public GraphPlanner(Configs graphConfig) { this.graphConfig = graphConfig; this.plannerConfig = PlannerConfig.create(this.graphConfig); logger.debug("planner config: " + this.plannerConfig); this.optPlanner = createRelOptPlanner(this.plannerConfig); - this.rexBuilder = new GraphRexBuilder(new GraphTypeFactoryImpl(graphConfig)); + this.rexBuilder = rexBuilderFactory.apply(graphConfig); this.idGenerator = new AtomicLong(FrontendConfig.FRONTEND_SERVER_ID.get(graphConfig)); }