From 0d24045159756f0ad0f42ef8589588da6c8dcc5a Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 12 Oct 2023 16:27:41 +0800 Subject: [PATCH] [GIE Compiler] classify filters into label equals or unique key equals or others --- .../ir/rel/graph/GraphLogicalSource.java | 15 +- .../common/ir/rex/RexNodeTypeRefresher.java | 62 ---- .../ir/runtime/ffi/FfiPhysicalBuilder.java | 3 +- .../ir/runtime/ffi/RelToFfiConverter.java | 144 ++++++--- .../ir/runtime/proto/RexToProtoConverter.java | 25 +- .../common/ir/runtime/proto/Utils.java | 49 ++- .../common/ir/tools/AliasInference.java | 4 +- .../common/ir/tools/GraphBuilder.java | 289 ++++++++++++++++-- .../common/ir/tools/GraphPlanner.java | 7 +- .../graphscope/common/ir/tools/Utils.java | 22 ++ .../graphscope/common/ir/FilterTest.java | 161 ++++++++++ .../common/ir/runtime/FfiLogicalPlanTest.java | 90 ++++++ .../common/ir/runtime/RexToProtoTest.java | 2 +- .../test/resources/ffi_logical_plan_6.json | 204 +++++++++++++ .../test/resources/ffi_logical_plan_7.json | 258 ++++++++++++++++ .../test/resources/ffi_logical_plan_8.json | 182 +++++++++++ 16 files changed, 1373 insertions(+), 144 deletions(-) delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java create mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json create mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json create mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java index cc7e88b3694a..2d6fce56a41c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java @@ -22,12 +22,15 @@ import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rex.RexNode; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import java.util.Objects; public class GraphLogicalSource extends AbstractBindableTableScan { private final GraphOpt.Source opt; + private @Nullable RexNode uniqueKeyFilters; protected GraphLogicalSource( GraphOptCluster cluster, @@ -54,6 +57,16 @@ public GraphOpt.Source getOpt() { @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("opt", getOpt()); + return super.explainTerms(pw) + .item("opt", getOpt()) + .itemIf("uniqueKeyFilters", uniqueKeyFilters, uniqueKeyFilters != null); + } + + public void setUniqueKeyFilters(RexNode uniqueKeyFilters) { + this.uniqueKeyFilters = Objects.requireNonNull(uniqueKeyFilters); + } + + public @Nullable RexNode getUniqueKeyFilters() { + return uniqueKeyFilters; } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java deleted file mode 100644 index 458adb42abc6..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.rex; - -import com.alibaba.graphscope.common.ir.tools.GraphRexBuilder; - -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.*; - -public class RexNodeTypeRefresher extends RexVisitorImpl { - private final RelDataType newType; - private final GraphRexBuilder rexBuilder; - - public RexNodeTypeRefresher(RelDataType newType, GraphRexBuilder rexBuilder) { - super(false); - this.newType = newType; - this.rexBuilder = rexBuilder; - } - - @Override - public RexNode visitCall(RexCall call) { - return call; - } - - @Override - public RexNode visitLiteral(RexLiteral literal) { - return literal; - } - - @Override - public RexNode visitInputRef(RexInputRef inputRef) { - return inputRef; - } - - @Override - public RexNode visitDynamicParam(RexDynamicParam dynamicParam) { - if (dynamicParam instanceof RexGraphDynamicParam) { - return visitGraphDynamicParam((RexGraphDynamicParam) dynamicParam); - } else { - return dynamicParam; - } - } - - private RexNode visitGraphDynamicParam(RexGraphDynamicParam graphDynamicParam) { - return rexBuilder.makeGraphDynamicParam( - newType, graphDynamicParam.getName(), graphDynamicParam.getIndex()); - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java index 9216217d96da..b05ce9b46651 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java @@ -68,7 +68,8 @@ public FfiPhysicalBuilder( Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan, PlanPointer planPointer) { super( logicalPlan, - new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId()))); + new GraphRelShuttleWrapper( + new RelToFfiConverter(irMeta.getSchema().isColumnId(), graphConfig))); this.graphConfig = graphConfig; this.irMeta = irMeta; this.planPointer = Objects.requireNonNull(planPointer); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java index f9efd30c80c9..02332052ab5f 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.common.ir.runtime.ffi; +import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.intermediate.ArgUtils; import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate; import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject; @@ -31,8 +32,10 @@ import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter; import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode; import com.alibaba.graphscope.common.ir.tools.AliasInference; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.alibaba.graphscope.common.ir.type.GraphNameOrId; import com.alibaba.graphscope.common.ir.type.GraphProperty; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.common.jna.IrCoreLibrary; @@ -49,17 +52,13 @@ import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexVariable; +import org.apache.calcite.rex.*; import org.apache.calcite.sql.SqlKind; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -72,9 +71,11 @@ public class RelToFfiConverter implements GraphRelShuttle { private static final Logger logger = LoggerFactory.getLogger(RelToFfiConverter.class); private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE; private final boolean isColumnId; + private final RexBuilder rexBuilder; - public RelToFfiConverter(boolean isColumnId) { + public RelToFfiConverter(boolean isColumnId, Configs configs) { this.isColumnId = isColumnId; + this.rexBuilder = GraphPlanner.rexBuilderFactory.apply(configs); } @Override @@ -205,7 +206,9 @@ public RelNode visit(GraphLogicalMultiMatch match) { @Override public RelNode visit(LogicalFilter logicalFilter) { OuterExpression.Expression exprProto = - logicalFilter.getCondition().accept(new RexToProtoConverter(true, isColumnId)); + logicalFilter + .getCondition() + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); Pointer ptrFilter = LIB.initSelectOperator(); checkFfiResult( LIB.setSelectPredicatePb( @@ -219,7 +222,9 @@ public PhysicalNode visit(GraphLogicalProject project) { List fields = project.getRowType().getFieldList(); for (int i = 0; i < project.getProjects().size(); ++i) { OuterExpression.Expression expression = - project.getProjects().get(i).accept(new RexToProtoConverter(true, isColumnId)); + project.getProjects() + .get(i) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); int aliasId = fields.get(i).getIndex(); FfiAlias.ByValue ffiAlias = (aliasId == AliasInference.DEFAULT_ID) @@ -260,7 +265,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { RexGraphVariable.class, var.getClass()); OuterExpression.Expression expr = - var.accept(new RexToProtoConverter(true, isColumnId)); + var.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); int aliasId; if (i >= fields.size() || (aliasId = fields.get(i).getIndex()) == AliasInference.DEFAULT_ID) { @@ -283,7 +288,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { field.getName(), field.getType()); OuterExpression.Variable exprVar = - rexVar.accept(new RexToProtoConverter(true, isColumnId)) + rexVar.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -305,7 +310,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { OuterExpression.Variable var = groupKeys .get(i) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); int aliasId = fields.get(i).getIndex(); @@ -339,7 +344,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { operands.get(0).getClass()); OuterExpression.Variable var = operands.get(0) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -369,7 +374,7 @@ public PhysicalNode visit(GraphLogicalSort sort) { for (int i = 0; i < collations.size(); ++i) { RexGraphVariable expr = ((GraphFieldCollation) collations.get(i)).getVariable(); OuterExpression.Variable var = - expr.accept(new RexToProtoConverter(true, isColumnId)) + expr.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -405,13 +410,13 @@ public PhysicalNode visit(LogicalJoin join) { OuterExpression.Variable leftVar = leftRightVars .get(0) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); OuterExpression.Variable rightVar = leftRightVars .get(1) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -451,7 +456,10 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) { }); if (ObjectUtils.isNotEmpty(tableScan.getFilters())) { OuterExpression.Expression expression = - tableScan.getFilters().get(0).accept(new RexToProtoConverter(true, isColumnId)); + tableScan + .getFilters() + .get(0) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); checkFfiResult( LIB.setParamsPredicatePb( params, new FfiPbPointer.ByValue(expression.toByteArray()))); @@ -460,46 +468,85 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) { } private @Nullable Pointer ffiIndexPredicates(GraphLogicalSource source) { - ImmutableList filters = source.getFilters(); - if (ObjectUtils.isEmpty(filters)) return null; - // decomposed by OR - List disJunctions = RelOptUtil.disjunctions(filters.get(0)); - List literals = new ArrayList<>(); - for (RexNode rexNode : disJunctions) { - if (!isIdEqualsLiteral(rexNode, literals)) { - return null; - } - } + RexNode uniqueKeyFilters = source.getUniqueKeyFilters(); + if (uniqueKeyFilters == null) return null; + // 'within' operator in index predicate is unsupported in ir core, here just expand it to + // 'or' + // i.e. '~id within [1, 2]' -> '~id == 1 or ~id == 2' + RexNode expandSearch = RexUtil.expandSearch(this.rexBuilder, null, uniqueKeyFilters); + List disjunctions = RelOptUtil.disjunctions(expandSearch); Pointer ptrIndex = LIB.initIndexPredicate(); - for (RexNode literal : literals) { - FfiProperty.ByValue property = new FfiProperty.ByValue(); - property.opt = FfiPropertyOpt.Id; - checkFfiResult( - LIB.orEquivPredicate(ptrIndex, property, Utils.ffiConst((RexLiteral) literal))); + for (RexNode disjunction : disjunctions) { + if (disjunction instanceof RexCall) { + RexCall rexCall = (RexCall) disjunction; + switch (rexCall.getOperator().getKind()) { + case EQUALS: + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (left instanceof RexGraphVariable + && (right instanceof RexLiteral + || right instanceof RexDynamicParam)) { + LIB.orEquivPredicate( + ptrIndex, + getFfiProperty(((RexGraphVariable) left).getProperty()), + getFfiConst(right)); + break; + } else if (right instanceof RexGraphVariable + && (left instanceof RexLiteral + || left instanceof RexDynamicParam)) { + LIB.orEquivPredicate( + ptrIndex, + getFfiProperty(((RexGraphVariable) right).getProperty()), + getFfiConst(left)); + break; + } + default: + throw new IllegalArgumentException( + "can not convert unique key filter pattern=" + + rexCall + + " to ir core index predicate"); + } + } else { + throw new IllegalArgumentException( + "invalid unique key filter pattern=" + disjunction); + } } - // remove index predicates from filter conditions - source.setFilters(ImmutableList.of()); return ptrIndex; } - // i.e. a.~id == 10 - private boolean isIdEqualsLiteral(RexNode rexNode, List literal) { - if (rexNode.getKind() != SqlKind.EQUALS) return false; - List operands = ((RexCall) rexNode).getOperands(); - return isGlobalId(operands.get(0)) && isLiteral(operands.get(1), literal) - || isGlobalId(operands.get(1)) && isLiteral(operands.get(0), literal); + private FfiProperty.ByValue getFfiProperty(GraphProperty property) { + Preconditions.checkArgument(property != null, "unique key should not be null"); + FfiProperty.ByValue ffiProperty = new FfiProperty.ByValue(); + switch (property.getOpt()) { + case ID: + ffiProperty.opt = FfiPropertyOpt.Id; + break; + case KEY: + ffiProperty.opt = FfiPropertyOpt.Key; + ffiProperty.key = getFfiNameOrId(property.getKey()); + break; + default: + throw new IllegalArgumentException( + "can not convert property=" + property + " to ffi property"); + } + return ffiProperty; } - // i.e. a.~id - private boolean isGlobalId(RexNode rexNode) { - return rexNode instanceof RexGraphVariable - && ((RexGraphVariable) rexNode).getProperty().getOpt() == GraphProperty.Opt.ID; + private FfiNameOrId.ByValue getFfiNameOrId(GraphNameOrId nameOrId) { + switch (nameOrId.getOpt()) { + case NAME: + return ArgUtils.asNameOrId(nameOrId.getName()); + case ID: + default: + return ArgUtils.asNameOrId(nameOrId.getId()); + } } - private boolean isLiteral(RexNode rexNode, List literal) { - boolean isLiteral = rexNode.getKind() == SqlKind.LITERAL; - if (isLiteral) literal.add(rexNode); - return isLiteral; + private FfiConst.ByValue getFfiConst(RexNode rexNode) { + if (rexNode instanceof RexLiteral) { + return Utils.ffiConst((RexLiteral) rexNode); + } + throw new IllegalArgumentException("cannot convert rexNode=" + rexNode + " to ffi const"); } private List range(RexNode offset, RexNode fetch) { @@ -592,7 +639,8 @@ private void addFilterToFfiBinder(Pointer ptrSentence, AbstractBindableTableScan List filters = tableScan.getFilters(); if (ObjectUtils.isNotEmpty(filters)) { OuterExpression.Expression exprProto = - filters.get(0).accept(new RexToProtoConverter(true, isColumnId)); + filters.get(0) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); Pointer ptrFilter = LIB.initSelectOperator(); checkFfiResult( LIB.setSelectPredicatePb( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java index 3e57e4e77d3f..220074c03e5d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java @@ -27,6 +27,7 @@ import org.apache.calcite.rex.*; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.util.Sarg; import java.util.List; @@ -35,10 +36,12 @@ */ public class RexToProtoConverter extends RexVisitorImpl { private final boolean isColumnId; + private final RexBuilder rexBuilder; - public RexToProtoConverter(boolean deep, boolean isColumnId) { + public RexToProtoConverter(boolean deep, boolean isColumnId, RexBuilder rexBuilder) { super(deep); this.isColumnId = isColumnId; + this.rexBuilder = rexBuilder; } @Override @@ -166,6 +169,26 @@ private OuterExpression.Expression visitIsNullOperator(RexNode operand) { } private OuterExpression.Expression visitBinaryOperator(RexCall call) { + if (call.getOperator().getKind() == SqlKind.SEARCH) { + // ir core can not support continuous ranges in a search operator, here expand it to + // compositions of 'and' or 'or', + // i.e. a.age SEARCH [[1, 10]] -> a.age >= 1 and a.age <= 10 + RexNode left = call.getOperands().get(0); + RexNode right = call.getOperands().get(1); + RexLiteral literal = null; + if (left instanceof RexLiteral) { + literal = (RexLiteral) left; + } else if (right instanceof RexLiteral) { + literal = (RexLiteral) right; + } + if (literal != null && literal.getValue() instanceof Sarg) { + Sarg sarg = (Sarg) literal.getValue(); + // search continuous ranges + if (!sarg.isPoints()) { + call = (RexCall) RexUtil.expandSearch(this.rexBuilder, null, call); + } + } + } SqlOperator operator = call.getOperator(); OuterExpression.Expression.Builder exprBuilder = OuterExpression.Expression.newBuilder(); // left-associative diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index 1245a82bb6d6..96842620312e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -17,7 +17,10 @@ package com.alibaba.graphscope.common.ir.runtime.proto; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; -import com.alibaba.graphscope.common.ir.type.*; +import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.alibaba.graphscope.common.ir.type.GraphNameOrId; +import com.alibaba.graphscope.common.ir.type.GraphProperty; +import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.gaia.proto.Common; import com.alibaba.graphscope.gaia.proto.DataType; import com.alibaba.graphscope.gaia.proto.GraphAlgebra; @@ -31,6 +34,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.Sarg; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +48,43 @@ public abstract class Utils { private static final Logger logger = LoggerFactory.getLogger(Utils.class); public static final Common.Value protoValue(RexLiteral literal) { + if (literal.getTypeName() == SqlTypeName.SARG) { + Sarg sarg = literal.getValueAs(Sarg.class); + if (!sarg.isPoints()) { + throw new UnsupportedOperationException( + "can not convert continuous ranges to ir core array, sarg=" + sarg); + } + List values = + com.alibaba.graphscope.common.ir.tools.Utils.getValuesAsList(sarg); + switch (literal.getType().getSqlTypeName()) { + case INTEGER: + Common.I32Array.Builder i32Array = Common.I32Array.newBuilder(); + values.forEach(value -> i32Array.addItem(((Number) value).intValue())); + return Common.Value.newBuilder().setI32Array(i32Array).build(); + case BIGINT: + Common.I64Array.Builder i64Array = Common.I64Array.newBuilder(); + values.forEach(value -> i64Array.addItem(((Number) value).longValue())); + return Common.Value.newBuilder().setI64Array(i64Array).build(); + case CHAR: + Common.StringArray.Builder stringArray = Common.StringArray.newBuilder(); + values.forEach( + value -> + stringArray.addItem( + (value instanceof NlsString) + ? ((NlsString) value).getValue() + : (String) value)); + return Common.Value.newBuilder().setStrArray(stringArray).build(); + case DECIMAL: + case FLOAT: + case DOUBLE: + Common.DoubleArray.Builder doubleArray = Common.DoubleArray.newBuilder(); + values.forEach(value -> doubleArray.addItem(((Number) value).doubleValue())); + return Common.Value.newBuilder().setF64Array(doubleArray).build(); + default: + throw new UnsupportedOperationException( + "can not convert sarg=" + sarg + " ir core array"); + } + } switch (literal.getType().getSqlTypeName()) { case NULL: return Common.Value.newBuilder().setNone(Common.None.newBuilder().build()).build(); @@ -70,7 +111,6 @@ public static final Common.Value protoValue(RexLiteral literal) { .setF64(((Number) literal.getValue()).doubleValue()) .build(); default: - // TODO: support int/double/string array throw new UnsupportedOperationException( "literal type " + literal.getTypeName() + " is unsupported yet"); } @@ -180,8 +220,11 @@ public static final OuterExpression.ExprOpr protoOperator(SqlOperator operator) return OuterExpression.ExprOpr.newBuilder() .setLogical(OuterExpression.Logical.ISNULL) .build(); + case SEARCH: + return OuterExpression.ExprOpr.newBuilder() + .setLogical(OuterExpression.Logical.WITHIN) + .build(); default: - // TODO: support IN and NOT_IN throw new UnsupportedOperationException( "operator type=" + operator.getKind() diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java index 33eb338b677c..177f83d17b61 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java @@ -176,7 +176,9 @@ public static final Set getUniqueAliasList(@Nullable RelNode input, bool while (!inputsQueue.isEmpty()) { RelNode cur = inputsQueue.remove(0); for (RelDataTypeField field : cur.getRowType().getFieldList()) { - uniqueNames.add(field.getName()); + if (field.getName() != null && field.getName() != DEFAULT_NAME) { + uniqueNames.add(field.getName()); + } } if (removeAlias(cur)) { break; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index 6f3d9b3e2f98..ff89811ef4a1 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -35,11 +35,9 @@ import com.alibaba.graphscope.common.ir.rex.*; import com.alibaba.graphscope.common.ir.rex.RexCallBinding; import com.alibaba.graphscope.common.ir.tools.config.*; -import com.alibaba.graphscope.common.ir.type.GraphNameOrId; -import com.alibaba.graphscope.common.ir.type.GraphPathType; -import com.alibaba.graphscope.common.ir.type.GraphProperty; -import com.alibaba.graphscope.common.ir.type.GraphSchemaType; +import com.alibaba.graphscope.common.ir.type.*; import com.alibaba.graphscope.gremlin.Utils; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -61,11 +59,13 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Pair; +import org.apache.calcite.util.Sarg; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; import java.math.BigDecimal; import java.util.*; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -581,7 +581,14 @@ private RexNode call_(SqlOperator operator, List operandList) { // derive return type RelDataType returnType = operator.inferReturnType(callBinding); // derive unknown types of operands - operandList = inferOperandTypes(operator, returnType, operandList); + operandList = + inferOperandTypes(operator, returnType, convertOperands(operator, operandList)); + final RexBuilder builder = cluster.getRexBuilder(); + return builder.makeCall(returnType, operator, operandList); + } + + // convert operands of the operator in some special cases + private List convertOperands(SqlOperator operator, List operandList) { if (operator.getKind() == SqlKind.EXTRACT) { RexNode intervalOperand = operandList.get(0); if (intervalOperand instanceof RexLiteral @@ -593,11 +600,11 @@ private RexNode call_(SqlOperator operator, List operandList) { getRexBuilder() .makeFlag(intervalType.getIntervalQualifier().getStartUnit())); newOperands.add(operandList.get(1)); - operandList = newOperands; + return newOperands; } } - final RexBuilder builder = cluster.getRexBuilder(); - return builder.makeCall(returnType, operator, operandList); + // todo: convert label names to ids + return operandList; } private List inferOperandTypes( @@ -612,10 +619,16 @@ private List inferOperandTypes( List typeInferredOperands = new ArrayList<>(operandList.size()); GraphRexBuilder rexBuilder = (GraphRexBuilder) this.getRexBuilder(); for (int i = 0; i < operandList.size(); ++i) { - typeInferredOperands.add( - operandList - .get(i) - .accept(new RexNodeTypeRefresher(newTypes[i], rexBuilder))); + RexNode rexNode = operandList.get(i); + if (rexNode instanceof RexGraphDynamicParam) { + RexGraphDynamicParam graphDynamicParam = (RexGraphDynamicParam) rexNode; + rexNode = + rexBuilder.makeGraphDynamicParam( + newTypes[i], + graphDynamicParam.getName(), + graphDynamicParam.getIndex()); + } + typeInferredOperands.add(rexNode); } return typeInferredOperands; } else { @@ -638,7 +651,8 @@ private boolean isCurrentSupported(SqlOperator operator) { || sqlKind == SqlKind.ARRAY_VALUE_CONSTRUCTOR || sqlKind == SqlKind.IS_NULL || sqlKind == SqlKind.IS_NOT_NULL - || sqlKind == SqlKind.EXTRACT; + || sqlKind == SqlKind.EXTRACT + || sqlKind == SqlKind.SEARCH; } @Override @@ -680,23 +694,245 @@ public GraphBuilder filter(Iterable conditions) { AliasInference.SIMPLE_NAME(AliasInference.DEFAULT_NAME), AliasInference.DEFAULT_ID)); // add condition into table scan - if (ObjectUtils.isEmpty(tableScan.getFilters())) { - tableScan.setFilters(ImmutableList.of(condition)); - } else { - ImmutableList.Builder listBuilder = new ImmutableList.Builder(); - listBuilder.addAll(tableScan.getFilters()).add(condition); - tableScan.setFilters( - ImmutableList.of( - RexUtil.composeConjunction( - this.getRexBuilder(), listBuilder.build()))); - } // pop the filter from the inner stack - replaceTop(tableScan); + replaceTop(fuseFilters(tableScan, condition)); } } return builder; } + private AbstractBindableTableScan fuseFilters( + AbstractBindableTableScan tableScan, RexNode condition) { + List labelValues = Lists.newArrayList(); + List uniqueKeyFilters = Lists.newArrayList(); + List extraFilters = Lists.newArrayList(); + classifyFilters(tableScan, condition, labelValues, uniqueKeyFilters, extraFilters); + if (!labelValues.isEmpty()) { + GraphLabelType labelType = + ((GraphSchemaType) tableScan.getRowType().getFieldList().get(0).getType()) + .getLabelType(); + List labelsToKeep = + labelType.getLabelsEntry().stream() + .filter(k -> labelValues.contains(k.getLabel())) + .map(k -> k.getLabel()) + .collect(Collectors.toList()); + Preconditions.checkArgument( + !labelsToKeep.isEmpty(), + "cannot find common labels between values= " + labelValues + " and label=", + labelType); + if (labelsToKeep.size() < labelType.getLabelsEntry().size()) { + GraphBuilder builder = + (GraphBuilder) + GraphPlanner.relBuilderFactory.create( + getCluster(), getRelOptSchema()); + LabelConfig newLabelConfig = new LabelConfig(false); + labelsToKeep.forEach(k -> newLabelConfig.addLabel(k)); + if (tableScan instanceof GraphLogicalSource) { + builder.source( + new SourceConfig( + ((GraphLogicalSource) tableScan).getOpt(), + newLabelConfig, + tableScan.getAliasName())); + } else if (tableScan instanceof GraphLogicalExpand) { + ((GraphBuilder) builder.push(tableScan.getInput(0))) + .expand( + new ExpandConfig( + ((GraphLogicalExpand) tableScan).getOpt(), + newLabelConfig, + tableScan.getAliasName())); + } else if (tableScan instanceof GraphLogicalGetV) { + ((GraphBuilder) builder.push(tableScan.getInput(0))) + .getV( + new GetVConfig( + ((GraphLogicalGetV) tableScan).getOpt(), + newLabelConfig, + tableScan.getAliasName())); + } + if (builder.size() > 0) { + // check if the property still exist after updating the label type + RexVisitor propertyChecker = + new RexVisitorImpl(true) { + @Override + public Void visitInputRef(RexInputRef inputRef) { + if (inputRef instanceof RexGraphVariable) { + RexGraphVariable variable = (RexGraphVariable) inputRef; + String[] splits = + variable.getName() + .split( + Pattern.quote( + AliasInference.DELIMITER)); + if (splits.length > 1) { + builder.variable(null, splits[1]); + } + } + return null; + } + }; + if (tableScan instanceof GraphLogicalSource) { + RexNode originalUniqueKeyFilters = + ((GraphLogicalSource) tableScan).getUniqueKeyFilters(); + if (originalUniqueKeyFilters != null) { + originalUniqueKeyFilters.accept(propertyChecker); + builder.filter(originalUniqueKeyFilters); + } + } + ImmutableList originalFilters = tableScan.getFilters(); + if (ObjectUtils.isNotEmpty(originalFilters)) { + originalFilters.forEach(k -> ((RexNode) k).accept(propertyChecker)); + builder.filter(originalFilters); + } + if (!extraFilters.isEmpty()) { + extraFilters.forEach(k -> k.accept(propertyChecker)); + } + tableScan = (AbstractBindableTableScan) builder.build(); + } + } + } + if (tableScan instanceof GraphLogicalSource && !uniqueKeyFilters.isEmpty()) { + GraphLogicalSource source = (GraphLogicalSource) tableScan; + Preconditions.checkArgument( + source.getUniqueKeyFilters() == null, + "can not add unique key filters if original is not empty"); + source.setUniqueKeyFilters( + RexUtil.composeDisjunction(this.getRexBuilder(), uniqueKeyFilters)); + } + if (!extraFilters.isEmpty()) { + ImmutableList originalFilters = tableScan.getFilters(); + if (ObjectUtils.isNotEmpty(originalFilters)) { + for (int i = 0; i < originalFilters.size(); ++i) { + extraFilters.add(i, (RexNode) originalFilters.get(i)); + } + } + tableScan.setFilters( + ImmutableList.of( + RexUtil.composeConjunction(this.getRexBuilder(), extraFilters))); + } + return tableScan; + } + + private void classifyFilters( + AbstractBindableTableScan tableScan, + RexNode condition, + List labelValues, + List uniqueKeyFilters, // unique key filters int the list are composed by 'OR' + List filters) { + List conjunctions = RelOptUtil.conjunctions(condition); + List filtersToRemove = Lists.newArrayList(); + for (RexNode conjunction : conjunctions) { + RexLiteral labelLiteral = isLabelEqualFilter(conjunction); + if (labelLiteral != null) { + filtersToRemove.add(conjunction); + labelValues.addAll( + com.alibaba.graphscope.common.ir.tools.Utils.getValuesAsList( + labelLiteral.getValueAs(Comparable.class))); + break; + } + } + if (tableScan instanceof GraphLogicalSource + && ((GraphLogicalSource) tableScan).getUniqueKeyFilters() == null) { + // try to extract unique key filters from the original condition + List disjunctions = RelOptUtil.disjunctions(condition); + for (RexNode disjunction : disjunctions) { + if (isUniqueKeyEqualFilter(disjunction)) { + filtersToRemove.add(disjunction); + uniqueKeyFilters.add(disjunction); + } + } + } + if (!filtersToRemove.isEmpty()) { + conjunctions.removeAll(filtersToRemove); + } + filters.addAll(conjunctions); + } + + // check the condition if it is the pattern of label equal filter, i.e. ~label = 'person' or + // ~label within ['person', 'software'] + // if it is then return the literal containing label values, otherwise null + private @Nullable RexLiteral isLabelEqualFilter(RexNode condition) { + if (condition instanceof RexCall) { + RexCall rexCall = (RexCall) condition; + SqlOperator operator = rexCall.getOperator(); + switch (operator.getKind()) { + case EQUALS: + case SEARCH: + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (left.getType() instanceof GraphLabelType && right instanceof RexLiteral) { + Comparable value = ((RexLiteral) right).getValue(); + // if Sarg is a continuous range then the filter is not the 'equal', i.e. + // ~label SEARCH [[1, 10]] which means ~label >= 1 and ~label <= 10 + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return null; + } + return (RexLiteral) right; + } else if (right.getType() instanceof GraphLabelType + && left instanceof RexLiteral) { + Comparable value = ((RexLiteral) left).getValue(); + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return null; + } + return (RexLiteral) left; + } + default: + return null; + } + } else { + return null; + } + } + + // check the condition if it is the pattern of unique key equal filter, i.e. ~id = 1 or ~id + // within [1, 2] + private boolean isUniqueKeyEqualFilter(RexNode condition) { + if (condition instanceof RexCall) { + RexCall rexCall = (RexCall) condition; + SqlOperator operator = rexCall.getOperator(); + switch (operator.getKind()) { + case EQUALS: + case SEARCH: + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (isUniqueKey(left) && isLiteralOrDynamicParams(right)) { + if (right instanceof RexLiteral) { + Comparable value = ((RexLiteral) right).getValue(); + // if Sarg is a continuous range then the filter is not the 'equal', + // i.e. ~id SEARCH [[1, 10]] which means ~id >= 1 and ~id <= 10 + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return false; + } + } + return true; + } else if (isUniqueKey(right) && isLiteralOrDynamicParams(left)) { + if (left instanceof RexLiteral) { + Comparable value = ((RexLiteral) left).getValue(); + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return false; + } + } + return true; + } + default: + return false; + } + } else { + return false; + } + } + + private boolean isUniqueKey(RexNode rexNode) { + // todo: support primary keys + if (rexNode instanceof RexGraphVariable) { + RexGraphVariable variable = (RexGraphVariable) rexNode; + return variable.getProperty() != null + && variable.getProperty().getOpt() == GraphProperty.Opt.ID; + } + return false; + } + + private boolean isLiteralOrDynamicParams(RexNode node) { + return node instanceof RexLiteral || node instanceof RexDynamicParam; + } + // return the top node if its type is Filter, otherwise null private Filter topFilter() { if (this.size() > 0 && this.peek() instanceof Filter) { @@ -1178,6 +1414,11 @@ public RexNode equals(RexNode operand0, RexNode operand1) { return this.call(GraphStdOperatorTable.EQUALS, operand0, operand1); } + @Override + public RexNode not(RexNode operand) { + return this.call(GraphStdOperatorTable.NOT, operand); + } + /** * {@code FilterIntoJoinRule} will put a new project on top of the join, but it is not necessary and will lead to extra cost, here we override it and do nothing to skip the extra project * @param castRowType diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 80e85957ee58..347c182ee35c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -61,6 +61,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; /** * A unified structure to build {@link PlannerInstance} which can further build logical and physical plan from an antlr tree @@ -72,16 +73,18 @@ public class GraphPlanner { private final RelOptPlanner optPlanner; private final RexBuilder rexBuilder; private final AtomicLong idGenerator; - private static final RelBuilderFactory relBuilderFactory = + public static final RelBuilderFactory relBuilderFactory = (RelOptCluster cluster, @Nullable RelOptSchema schema) -> GraphBuilder.create(null, (GraphOptCluster) cluster, schema); + public static final Function rexBuilderFactory = + (Configs configs) -> new GraphRexBuilder(new GraphTypeFactoryImpl(configs)); public GraphPlanner(Configs graphConfig) { this.graphConfig = graphConfig; this.plannerConfig = PlannerConfig.create(this.graphConfig); logger.debug("planner config: " + this.plannerConfig); this.optPlanner = createRelOptPlanner(this.plannerConfig); - this.rexBuilder = new GraphRexBuilder(new GraphTypeFactoryImpl(graphConfig)); + this.rexBuilder = rexBuilderFactory.apply(graphConfig); this.idGenerator = new AtomicLong(FrontendConfig.FRONTEND_SERVER_ID.get(graphConfig)); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java index 701cc3ebb337..ea970934c9fd 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java @@ -16,7 +16,9 @@ package com.alibaba.graphscope.common.ir.tools; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.apache.calcite.rel.RelNode; @@ -24,6 +26,8 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelRecordType; import org.apache.calcite.rel.type.StructKind; +import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.Sarg; import java.util.List; import java.util.Set; @@ -53,4 +57,22 @@ public static RelDataType getOutputType(RelNode topNode) { .collect(Collectors.toList()); return new RelRecordType(StructKind.FULLY_QUALIFIED, dedup); } + + public static List getValuesAsList(Comparable value) { + ImmutableList.Builder valueBuilder = ImmutableList.builder(); + if (value instanceof NlsString) { + valueBuilder.add(((NlsString) value).getValue()); + } else if (value instanceof Sarg) { + Sarg sarg = (Sarg) value; + if (sarg.isPoints()) { + Set> rangeSets = sarg.rangeSet.asRanges(); + for (Range range : rangeSets) { + valueBuilder.addAll(getValuesAsList(range.lowerEndpoint())); + } + } + } else { + valueBuilder.add(value); + } + return valueBuilder.build(); + } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/FilterTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/FilterTest.java index 0f20d63844a8..d623a3fdb946 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/FilterTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/FilterTest.java @@ -22,9 +22,12 @@ import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.tools.config.LabelConfig; import com.alibaba.graphscope.common.ir.tools.config.SourceConfig; +import com.alibaba.graphscope.common.ir.type.GraphProperty; +import com.google.common.collect.ImmutableList; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.SqlTypeName; @@ -237,4 +240,162 @@ public void and_2_test() { + " fusedFilter=[[AND(>(DEFAULT.age, 20), <(DEFAULT.age, 30))]], opt=[VERTEX])", filter.explain().trim()); } + + // g.V().hasLabel('person') + @Test + public void label_equals_1_test() { + GraphBuilder builder = Utils.mockGraphBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal("person"))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}], alias=[DEFAULT]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + // g.V().hasLabel('XX'), 'XX' is not a valid label + @Test + public void label_equals_2_test() { + try { + GraphBuilder builder = Utils.mockGraphBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal("XX"))) + .build(); + } catch (IllegalArgumentException e) { + Assert.assertEquals( + "cannot find common labels between values= [XX] and label=" + + " [[VertexLabel(software), VertexLabel(person)]]", + e.getMessage()); + return; + } + Assert.fail(); + } + + // g.V().hasLabel('person').hasLabel('software') + @Test + public void label_equals_3_test() { + try { + GraphBuilder builder = Utils.mockGraphBuilder(); + RelNode node = + builder.source( + new SourceConfig( + GraphOpt.Source.VERTEX, + new LabelConfig(false).addLabel("person"))) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal("software"))) + .build(); + } catch (IllegalArgumentException e) { + Assert.assertEquals( + "cannot find common labels between values= [software] and label=" + + " [[VertexLabel(person)]]", + e.getMessage()); + return; + } + Assert.fail(); + } + + // g.V().has('age', 10).hasLabel('person') + @Test + public void label_equals_4_test() { + GraphBuilder builder = Utils.mockGraphBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, "age"), + builder.literal(10))) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal("person"))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}], alias=[DEFAULT]," + + " fusedFilter=[[=(DEFAULT.age, 10)]], opt=[VERTEX])", + node.explain().trim()); + } + + // g.V().has('age', 10).hasLabel('software'), property 'age' not exist in label='software' + @Test + public void label_equals_5_test() { + try { + GraphBuilder builder = Utils.mockGraphBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, "age"), + builder.literal(10))) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal("software"))) + .build(); + } catch (IllegalArgumentException e) { + Assert.assertEquals( + "{property=age} not found; expected properties are: [id, name, lang," + + " creationDate]", + e.getMessage()); + return; + } + Assert.fail(); + } + + // ~label within ['person'] + @Test + public void label_within_1_test() { + GraphBuilder builder = Utils.mockGraphBuilder(); + RexBuilder rexBuilder = builder.getRexBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + rexBuilder.makeIn( + builder.variable(null, GraphProperty.LABEL_KEY), + ImmutableList.of(builder.literal("person")))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}], alias=[DEFAULT]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + // g.V().hasId(1, 2, 3) + @Test + public void unique_key_within_test() { + GraphBuilder builder = Utils.mockGraphBuilder(); + RexBuilder rexBuilder = builder.getRexBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + rexBuilder.makeIn( + builder.variable(null, GraphProperty.ID_KEY), + ImmutableList.of( + builder.literal(1), + builder.literal(2), + builder.literal(3)))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX], uniqueKeyFilters=[SEARCH(DEFAULT.~id," + + " Sarg[1, 2, 3])])", + node.explain().trim()); + } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java index 1974ed4e3773..1d5827efea14 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java @@ -24,11 +24,13 @@ import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.common.ir.tools.config.*; +import com.alibaba.graphscope.common.ir.type.GraphProperty; import com.alibaba.graphscope.common.utils.FileUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.junit.Assert; import org.junit.Test; @@ -198,6 +200,94 @@ public void logical_plan_5_test() throws Exception { } } + // test conversion from search operator to within operator in ir core + // a.age SEARCH [1, 2, 3] -> a.age within [1, 2, 3] + @Test + public void logical_plan_6_test() throws Exception { + GraphBuilder builder = Utils.mockGraphBuilder(); + RexBuilder rexBuilder = builder.getRexBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + rexBuilder.makeIn( + builder.variable(null, "age"), + ImmutableList.of( + builder.literal(1), + builder.literal(2), + builder.literal(3)))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[1, 2, 3])]]," + + " opt=[VERTEX])", + node.explain().trim()); + try (PhysicalBuilder ffiBuilder = + new FfiPhysicalBuilder( + getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(node))) { + ffiBuilder.build(); + Assert.assertEquals( + FileUtils.readJsonFromResource("ffi_logical_plan_6.json"), + ffiBuilder.explain()); + } + } + + // test conversion from search continuous ranges to compositions of 'and' in ir core + // a.age SEARCH [[1..10]] -> a.age >= 1 and a.age <= 10 + @Test + public void logical_plan_7_test() throws Exception { + GraphBuilder builder = Utils.mockGraphBuilder(); + RexBuilder rexBuilder = builder.getRexBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + rexBuilder.makeBetween( + builder.variable(null, "age"), + builder.literal(1), + builder.literal(10))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[[1..10]])]]," + + " opt=[VERTEX])", + node.explain().trim()); + try (PhysicalBuilder ffiBuilder = + new FfiPhysicalBuilder( + getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(node))) { + ffiBuilder.build(); + Assert.assertEquals( + FileUtils.readJsonFromResource("ffi_logical_plan_7.json"), + ffiBuilder.explain()); + } + } + + // test conversion of index predicate + // ~id SEARCH [1, 2] -> ~id == 1 or ~id == 2 + @Test + public void logical_plan_8_test() throws Exception { + GraphBuilder builder = Utils.mockGraphBuilder(); + RexBuilder rexBuilder = builder.getRexBuilder(); + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .filter( + rexBuilder.makeIn( + builder.variable(null, GraphProperty.ID_KEY), + ImmutableList.of(builder.literal(1), builder.literal(2)))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX], uniqueKeyFilters=[SEARCH(DEFAULT.~id," + + " Sarg[1, 2])])", + node.explain().trim()); + try (PhysicalBuilder ffiBuilder = + new FfiPhysicalBuilder( + getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(node))) { + ffiBuilder.build(); + Assert.assertEquals( + FileUtils.readJsonFromResource("ffi_logical_plan_8.json"), + ffiBuilder.explain()); + } + } + private Configs getMockGraphConfig() { return new Configs(ImmutableMap.of("servers", "1", "workers", "1")); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java index 311f304f318a..191e112a274c 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java @@ -50,7 +50,7 @@ public void test_expression_with_brace() throws Exception { GraphStdOperatorTable.MINUS, builder.literal(1), builder.variable("a", "age"))); - RexToProtoConverter converter = new RexToProtoConverter(true, false); + RexToProtoConverter converter = new RexToProtoConverter(true, false, Utils.rexBuilder); Assert.assertEquals( FileUtils.readJsonFromResource("proto/expression_with_brace.json"), JsonFormat.printer().print(braceExpr.accept(converter))); diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json new file mode 100644 index 000000000000..312cb941531e --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json @@ -0,0 +1,204 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": null, + "params": { + "tables": [ + { + "item": { + "Id": 0 + } + }, + { + "item": { + "Id": 1 + } + } + ], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": { + "operators": [ + { + "node_type": { + "type": { + "DataType": 1 + } + }, + "item": { + "Var": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "age" + } + } + } + }, + "node_type": { + "type": { + "DataType": 1 + } + } + } + } + }, + { + "node_type": { + "type": { + "DataType": 0 + } + }, + "item": { + "Logical": 6 + } + }, + { + "node_type": { + "type": { + "DataType": 1 + } + }, + "item": { + "Const": { + "item": { + "I32Array": { + "item": [ + 1, + 2, + 3 + ] + } + } + } + } + } + ] + }, + "sample_ratio": 1.0, + "extra": {} + }, + "idx_predicate": null, + "is_count_only": false, + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 0, + "graph_data_type": [ + { + "label": { + "label": 1, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "lang" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "creationDate" + } + }, + "type": 12 + } + ] + }, + { + "label": { + "label": 0, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "age" + } + }, + "type": 1 + } + ] + } + ] + } + } + }, + "alias": -1 + } + } + } + }, + "children": [ + 1 + ] + }, + { + "opr": { + "opr": { + "Sink": { + "tags": [], + "sink_target": { + "inner": { + "SinkDefault": { + "id_name_mappings": [] + } + } + } + } + } + }, + "children": [] + } + ], + "roots": [ + 0 + ] +} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json new file mode 100644 index 000000000000..f83d67341255 --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json @@ -0,0 +1,258 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": null, + "params": { + "tables": [ + { + "item": { + "Id": 0 + } + }, + { + "item": { + "Id": 1 + } + } + ], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": { + "operators": [ + { + "node_type": { + "type": { + "DataType": 1 + } + }, + "item": { + "Var": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "age" + } + } + } + }, + "node_type": { + "type": { + "DataType": 1 + } + } + } + } + }, + { + "node_type": { + "type": { + "DataType": 0 + } + }, + "item": { + "Logical": 5 + } + }, + { + "node_type": { + "type": { + "DataType": 1 + } + }, + "item": { + "Const": { + "item": { + "I32": 1 + } + } + } + }, + { + "node_type": { + "type": { + "DataType": 0 + } + }, + "item": { + "Logical": 10 + } + }, + { + "node_type": { + "type": { + "DataType": 1 + } + }, + "item": { + "Var": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "age" + } + } + } + }, + "node_type": { + "type": { + "DataType": 1 + } + } + } + } + }, + { + "node_type": { + "type": { + "DataType": 0 + } + }, + "item": { + "Logical": 3 + } + }, + { + "node_type": { + "type": { + "DataType": 1 + } + }, + "item": { + "Const": { + "item": { + "I32": 10 + } + } + } + } + ] + }, + "sample_ratio": 1.0, + "extra": {} + }, + "idx_predicate": null, + "is_count_only": false, + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 0, + "graph_data_type": [ + { + "label": { + "label": 1, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "lang" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "creationDate" + } + }, + "type": 12 + } + ] + }, + { + "label": { + "label": 0, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "age" + } + }, + "type": 1 + } + ] + } + ] + } + } + }, + "alias": -1 + } + } + } + }, + "children": [ + 1 + ] + }, + { + "opr": { + "opr": { + "Sink": { + "tags": [], + "sink_target": { + "inner": { + "SinkDefault": { + "id_name_mappings": [] + } + } + } + } + } + }, + "children": [] + } + ], + "roots": [ + 0 + ] +} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json new file mode 100644 index 000000000000..f57b0273796b --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json @@ -0,0 +1,182 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": null, + "params": { + "tables": [ + { + "item": { + "Id": 0 + } + }, + { + "item": { + "Id": 1 + } + } + ], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "sample_ratio": 1.0, + "extra": {} + }, + "idx_predicate": { + "or_predicates": [ + { + "predicates": [ + { + "key": { + "item": { + "Id": {} + } + }, + "value": { + "item": { + "I32": 1 + } + }, + "cmp": null + } + ] + }, + { + "predicates": [ + { + "key": { + "item": { + "Id": {} + } + }, + "value": { + "item": { + "I32": 2 + } + }, + "cmp": null + } + ] + } + ] + }, + "is_count_only": false, + "meta_data": { + "type": { + "type": { + "GraphType": { + "element_opt": 0, + "graph_data_type": [ + { + "label": { + "label": 1, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "lang" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "creationDate" + } + }, + "type": 12 + } + ] + }, + { + "label": { + "label": 0, + "src_label": null, + "dst_label": null + }, + "props": [ + { + "prop_id": { + "item": { + "Name": "id" + } + }, + "type": 2 + }, + { + "prop_id": { + "item": { + "Name": "name" + } + }, + "type": 4 + }, + { + "prop_id": { + "item": { + "Name": "age" + } + }, + "type": 1 + } + ] + } + ] + } + } + }, + "alias": -1 + } + } + } + }, + "children": [ + 1 + ] + }, + { + "opr": { + "opr": { + "Sink": { + "tags": [], + "sink_target": { + "inner": { + "SinkDefault": { + "id_name_mappings": [] + } + } + } + } + } + }, + "children": [] + } + ], + "roots": [ + 0 + ] +}