From 5231f7cbe35729ed35a7da327463ec7044e08f0b Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 14 Sep 2023 16:06:25 +0800 Subject: [PATCH 01/10] [GIE Compiler] refine return type of ARRAY_VALUE_CONSTRUCTOR and MAP_VALUE_CONSTRUCTOR --- .../operator/SqlArrayValueConstructor.java | 56 +++++++++++++++ .../rex/operator/SqlMapValueConstructor.java | 72 +++++++++++++++++++ .../common/ir/tools/GraphBuilder.java | 1 + .../ir/tools/GraphStdOperatorTable.java | 14 ++-- .../graphscope/common/ir/ExpressionTest.java | 55 ++++++++++++++ 5 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java new file mode 100644 index 000000000000..ef56370b4c65 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.rex.operator; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.fun.SqlMultisetValueConstructor; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; + +import java.util.List; + +public class SqlArrayValueConstructor extends SqlMultisetValueConstructor { + public SqlArrayValueConstructor() { + super("ARRAY", SqlKind.ARRAY_VALUE_CONSTRUCTOR); + } + + @Override + public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + List argTypes = opBinding.collectOperandTypes(); + RelDataType componentType; + if (argTypes.isEmpty()) { + componentType = typeFactory.createSqlType(SqlTypeName.ANY); + } else { + componentType = getComponentType(typeFactory, argTypes); + if (componentType == null) { + componentType = typeFactory.createSqlType(SqlTypeName.ANY); + } + } + return SqlTypeUtil.createArrayType(typeFactory, componentType, false); + } + + // operands of array value constructor can be any, even if empty, i.e [] + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + return true; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java new file mode 100644 index 000000000000..4bedfa411d26 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.rex.operator; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.fun.SqlMultisetValueConstructor; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.util.Pair; +import org.apache.calcite.util.Static; +import org.apache.calcite.util.Util; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +public class SqlMapValueConstructor extends SqlMultisetValueConstructor { + public SqlMapValueConstructor() { + super("MAP", SqlKind.MAP_VALUE_CONSTRUCTOR); + } + + public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + Pair type = + getComponentTypes(opBinding.getTypeFactory(), opBinding.collectOperandTypes()); + return SqlTypeUtil.createMapType(opBinding.getTypeFactory(), type.left, type.right, false); + } + + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + List argTypes = callBinding.collectOperandTypes(); + if (argTypes.size() % 2 > 0) { + throw callBinding.newValidationError(Static.RESOURCE.mapRequiresEvenArgCount()); + } + return true; + } + + private static Pair<@Nullable RelDataType, @Nullable RelDataType> getComponentTypes( + RelDataTypeFactory typeFactory, List argTypes) { + RelDataType leftType, rightType; + if (argTypes.isEmpty()) { + leftType = typeFactory.createSqlType(SqlTypeName.ANY); + rightType = typeFactory.createSqlType(SqlTypeName.ANY); + } else { + leftType = typeFactory.leastRestrictive(Util.quotientList(argTypes, 2, 0)); + rightType = typeFactory.leastRestrictive(Util.quotientList(argTypes, 2, 1)); + if (leftType == null) { + leftType = typeFactory.createSqlType(SqlTypeName.ANY); + } + if (rightType == null) { + rightType = typeFactory.createSqlType(SqlTypeName.ANY); + } + } + return Pair.of(leftType, rightType); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index 7cb79604a7a4..47408a28a6b1 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -635,6 +635,7 @@ private boolean isCurrentSupported(SqlOperator operator) { || (sqlKind == SqlKind.PROCEDURE_CALL) || (sqlKind == SqlKind.NOT) || sqlKind == SqlKind.ARRAY_VALUE_CONSTRUCTOR + || sqlKind == SqlKind.MAP_VALUE_CONSTRUCTOR || sqlKind == SqlKind.IS_NULL || sqlKind == SqlKind.IS_NOT_NULL || sqlKind == SqlKind.EXTRACT; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java index 1959fa4571fb..1b5402d39b6a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java @@ -18,6 +18,8 @@ import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; import com.alibaba.graphscope.common.ir.rex.operator.CaseOperator; +import com.alibaba.graphscope.common.ir.rex.operator.SqlArrayValueConstructor; +import com.alibaba.graphscope.common.ir.rex.operator.SqlMapValueConstructor; import org.apache.calcite.sql.*; import org.apache.calcite.sql.fun.SqlMonotonicBinaryOperator; @@ -208,15 +210,7 @@ public static final SqlFunction USER_DEFINED_PROCEDURE(StoredProcedureMeta meta) } // combine multiple expressions into a list - public static final SqlOperator ARRAY_VALUE_CONSTRUCTOR = - new SqlSpecialOperator( - "ARRAY_VALUE_CONSTRUCTOR", - SqlKind.ARRAY_VALUE_CONSTRUCTOR, - 0, - false, - ReturnTypes.explicit(SqlTypeName.ANY).andThen(SqlTypeTransforms.TO_ARRAY), - GraphInferTypes.FIRST_KNOWN, - OperandTypes.ANY); + public static final SqlOperator ARRAY_VALUE_CONSTRUCTOR = new SqlArrayValueConstructor(); public static final SqlFunction EXTRACT = new SqlFunction( @@ -226,4 +220,6 @@ public static final SqlFunction USER_DEFINED_PROCEDURE(StoredProcedureMeta meta) null, GraphOperandTypes.INTERVALINTERVAL_INTERVALDATETIME, SqlFunctionCategory.SYSTEM); + + public static final SqlOperator MAP_VALUE_CONSTRUCTOR = new SqlMapValueConstructor(); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java index e64f23570e52..40fabeda952b 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java @@ -169,6 +169,61 @@ public void unary_minus_test() { Assert.assertEquals("-(a.age)", node.toString()); } + // check the return type of the 'ARRAY_VALUE_CONSTRUCTOR', the return type inference strategy + // is: + // 1. derive the component type by finding the least restrictive type of the arguments + // 2. if the least restrictive type cannot be found, use 'ANY' type + // 3. derive the array type by the component type + @Test + public void array_value_constructor_test() { + RexNode node = + builder.call( + GraphStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + builder.literal(1), + builder.literal(2)); + Assert.assertEquals("INTEGER ARRAY", node.getType().toString()); + + node = + builder.call( + GraphStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + builder.literal(1), + builder.literal(2.0)); + Assert.assertEquals("DOUBLE ARRAY", node.getType().toString()); + + node = + builder.source(mockSourceConfig("a")) + .call( + GraphStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + builder.variable("a", "age"), + builder.literal(1)); + Assert.assertEquals("INTEGER ARRAY", node.getType().toString()); + + node = + builder.source(mockSourceConfig("a")) + .call( + GraphStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + builder.variable("a", "name"), + builder.literal(1)); + Assert.assertEquals("ANY ARRAY", node.getType().toString()); + } + + // check the return type of the 'MAP_VALUE_CONSTRUCTOR', the return type inference strategy is: + // 1. derive the key or value type by finding the least restrictive type of the arguments + // 2. if the least restrictive type cannot be found, use 'ANY' type + // 3. derive the map type by the key or value type + @Test + public void map_value_constructor_test() { + RexNode node = + builder.source(mockSourceConfig("a")) + .call( + GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + builder.literal("name"), + builder.variable("a", "name"), + builder.literal("age"), + builder.variable("a", "age")); + Assert.assertEquals("(CHAR(4), ANY) MAP", node.getType().toString()); + } + private SourceConfig mockSourceConfig(String alias) { return new SourceConfig( GraphOpt.Source.VERTEX, new LabelConfig(false).addLabel("person"), alias); From b5a91d26cc61d00fc8a149ac5d06919b4f2e0ec1 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 22 Sep 2023 10:46:49 +0800 Subject: [PATCH 02/10] [GIE Compiler] support V/E/hasId/hasLabel/Has Steps --- .../common/ir/meta/schema/GraphOptTable.java | 20 +- .../rel/graph/AbstractBindableTableScan.java | 11 +- .../ir/rel/graph/GraphLogicalSource.java | 24 +- .../common/ir/rex/RexNodeTypeRefresher.java | 62 ----- .../ir/runtime/ffi/RelToFfiConverter.java | 15 +- .../common/ir/runtime/proto/Utils.java | 30 +-- .../common/ir/tools/AliasInference.java | 4 +- .../common/ir/tools/GraphBuilder.java | 245 ++++++++++++++--- .../common/ir/tools/GraphPlanner.java | 3 +- .../common/ir/type/GraphLabelType.java | 182 +++++++++---- .../common/ir/type/GraphSchemaType.java | 97 ++++++- .../common/ir/type/GraphSchemaTypeList.java | 177 ------------ .../common/ir/type/GraphTypeFactoryImpl.java | 24 +- .../cypher/result/CypherRecordParser.java | 49 ++-- .../antlr4x/parser/GremlinAntlr4Parser.java | 49 ++++ .../antlr4x/visitor/ExpressionVisitor.java | 254 ++++++++++++++++++ .../antlr4x/visitor/GraphBuilderVisitor.java | 183 +++++++++++++ .../graphscope/common/ir/ExpressionTest.java | 63 ++++- .../gremlin/antlr4x/GraphBuilderTest.java | 39 +++ 19 files changed, 1112 insertions(+), 419 deletions(-) delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaTypeList.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/parser/GremlinAntlr4Parser.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java index 4c9fed3c3b16..d47111c41906 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java @@ -21,7 +21,6 @@ import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphLabelType; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; -import com.alibaba.graphscope.common.ir.type.GraphSchemaTypeList; import com.alibaba.graphscope.groot.common.schema.api.*; import org.apache.calcite.linq4j.tree.Expression; @@ -76,27 +75,32 @@ private RelDataType deriveType(GraphElement element) { } if (element instanceof GraphVertex) { GraphLabelType labelType = - (new GraphLabelType()).label(element.getLabel()).labelId(element.getLabelId()); + new GraphLabelType( + new GraphLabelType.Entry() + .label(element.getLabel()) + .labelId(element.getLabelId())); return new GraphSchemaType(GraphOpt.Source.VERTEX, labelType, fields); } else if (element instanceof GraphEdge) { GraphEdge edge = (GraphEdge) element; List relations = edge.getRelationList(); List fuzzyTypes = new ArrayList<>(); for (EdgeRelation relation : relations) { - GraphLabelType labelType = - (new GraphLabelType()) + GraphLabelType.Entry labelEntry = + new GraphLabelType.Entry() .label(element.getLabel()) .labelId(element.getLabelId()); GraphVertex src = relation.getSource(); GraphVertex dst = relation.getTarget(); - labelType.srcLabel(src.getLabel()).dstLabel(dst.getLabel()); - labelType.srcLabelId(src.getLabelId()).dstLabelId(dst.getLabelId()); - fuzzyTypes.add(new GraphSchemaType(GraphOpt.Source.EDGE, labelType, fields)); + labelEntry.srcLabel(src.getLabel()).dstLabel(dst.getLabel()); + labelEntry.srcLabelId(src.getLabelId()).dstLabelId(dst.getLabelId()); + fuzzyTypes.add( + new GraphSchemaType( + GraphOpt.Source.EDGE, new GraphLabelType(labelEntry), fields)); } ObjectUtils.requireNonEmpty(fuzzyTypes); return (fuzzyTypes.size() == 1) ? fuzzyTypes.get(0) - : GraphSchemaTypeList.create(fuzzyTypes); + : GraphSchemaType.create(fuzzyTypes, getRelOptSchema().getTypeFactory()); } else { throw new IllegalArgumentException("element should be vertex or edge"); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java index 81e55ce4e5c9..30b243693806 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java @@ -19,7 +19,6 @@ import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.AliasInference; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; -import com.alibaba.graphscope.common.ir.type.GraphSchemaTypeList; import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.GraphOptCluster; @@ -30,6 +29,7 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeFieldImpl; import org.apache.calcite.rel.type.RelRecordType; import org.apache.calcite.rex.RexNode; @@ -91,20 +91,17 @@ protected AbstractBindableTableScan( public RelDataType deriveRowType() { List tableTypes = new ArrayList<>(); List tables = ObjectUtils.requireNonEmpty(this.tableConfig.getTables()); + RelDataTypeFactory typeFactory = tables.get(0).getRelOptSchema().getTypeFactory(); for (RelOptTable table : tables) { GraphSchemaType type = (GraphSchemaType) table.getRowType(); // flat fuzzy labels to the list - if (type instanceof GraphSchemaTypeList) { - tableTypes.addAll((GraphSchemaTypeList) type); - } else { - tableTypes.add(type); - } + tableTypes.addAll(type.getSchemaTypes()); } ObjectUtils.requireNonEmpty(tableTypes); GraphSchemaType graphType = (tableTypes.size() == 1) ? tableTypes.get(0) - : GraphSchemaTypeList.create(tableTypes); + : GraphSchemaType.create(tableTypes, typeFactory); RelRecordType rowType = new RelRecordType( ImmutableList.of( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java index cc7e88b3694a..5d5b3bbb1c94 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java @@ -22,12 +22,16 @@ import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rex.RexNode; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import java.util.Objects; public class GraphLogicalSource extends AbstractBindableTableScan { private final GraphOpt.Source opt; + private @Nullable RexNode uniqueKeyFilters; + private @Nullable RexNode primaryKeyFilters; protected GraphLogicalSource( GraphOptCluster cluster, @@ -54,6 +58,24 @@ public GraphOpt.Source getOpt() { @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("opt", getOpt()); + return super.explainTerms(pw) + .item("opt", getOpt()) + .itemIf("uniqueKeyFilters", uniqueKeyFilters, uniqueKeyFilters != null); + } + + public void setUniqueKeyFilters(RexNode uniqueKeyFilters) { + this.uniqueKeyFilters = Objects.requireNonNull(uniqueKeyFilters); + } + + public void setPrimaryKeyFilters(RexNode primaryKeyFilters) { + this.primaryKeyFilters = Objects.requireNonNull(primaryKeyFilters); + } + + public @Nullable RexNode getUniqueKeyFilters() { + return uniqueKeyFilters; + } + + public @Nullable RexNode getPrimaryKeyFilters() { + return primaryKeyFilters; } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java deleted file mode 100644 index 458adb42abc6..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/RexNodeTypeRefresher.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.rex; - -import com.alibaba.graphscope.common.ir.tools.GraphRexBuilder; - -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.*; - -public class RexNodeTypeRefresher extends RexVisitorImpl { - private final RelDataType newType; - private final GraphRexBuilder rexBuilder; - - public RexNodeTypeRefresher(RelDataType newType, GraphRexBuilder rexBuilder) { - super(false); - this.newType = newType; - this.rexBuilder = rexBuilder; - } - - @Override - public RexNode visitCall(RexCall call) { - return call; - } - - @Override - public RexNode visitLiteral(RexLiteral literal) { - return literal; - } - - @Override - public RexNode visitInputRef(RexInputRef inputRef) { - return inputRef; - } - - @Override - public RexNode visitDynamicParam(RexDynamicParam dynamicParam) { - if (dynamicParam instanceof RexGraphDynamicParam) { - return visitGraphDynamicParam((RexGraphDynamicParam) dynamicParam); - } else { - return dynamicParam; - } - } - - private RexNode visitGraphDynamicParam(RexGraphDynamicParam graphDynamicParam) { - return rexBuilder.makeGraphDynamicParam( - newType, graphDynamicParam.getName(), graphDynamicParam.getIndex()); - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java index 0c051e1fc7b3..f9efd30c80c9 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java @@ -35,7 +35,6 @@ import com.alibaba.graphscope.common.ir.type.GraphLabelType; import com.alibaba.graphscope.common.ir.type.GraphProperty; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; -import com.alibaba.graphscope.common.ir.type.GraphSchemaTypeList; import com.alibaba.graphscope.common.jna.IrCoreLibrary; import com.alibaba.graphscope.common.jna.type.*; import com.alibaba.graphscope.gaia.proto.OuterExpression; @@ -442,7 +441,7 @@ private List getLeftRightVariables(RexNode condition) { private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) { Set uniqueLabelIds = - getGraphLabels(tableScan).stream() + getGraphLabels(tableScan).getLabelsEntry().stream() .map(k -> k.getLabelId()) .collect(Collectors.toSet()); Pointer params = LIB.initQueryParams(); @@ -570,7 +569,7 @@ private void addFfiBinder(Pointer ptrSentence, RelNode binder, boolean isTail) { private void addFilterToFfiBinder(Pointer ptrSentence, AbstractBindableTableScan tableScan) { Set uniqueLabelIds = - getGraphLabels(tableScan).stream() + getGraphLabels(tableScan).getLabelsEntry().stream() .map(k -> k.getLabelId()) .collect(Collectors.toSet()); // add labels as select operator @@ -602,20 +601,14 @@ private void addFilterToFfiBinder(Pointer ptrSentence, AbstractBindableTableScan } } - private List getGraphLabels(AbstractBindableTableScan tableScan) { + private GraphLabelType getGraphLabels(AbstractBindableTableScan tableScan) { List fields = tableScan.getRowType().getFieldList(); Preconditions.checkArgument( !fields.isEmpty() && fields.get(0).getType() instanceof GraphSchemaType, "data type of graph operators should be %s ", GraphSchemaType.class); GraphSchemaType schemaType = (GraphSchemaType) fields.get(0).getType(); - List labelTypes = new ArrayList<>(); - if (schemaType instanceof GraphSchemaTypeList) { - ((GraphSchemaTypeList) schemaType).forEach(k -> labelTypes.add(k.getLabelType())); - } else { - labelTypes.add(schemaType.getLabelType()); - } - return labelTypes; + return schemaType.getLabelType(); } private void checkFfiResult(FfiResult res) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index df8395834f38..8d049b15dec3 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -243,17 +243,10 @@ public static final DataType.IrDataType protoIrDataType( DataType.GraphDataType.Builder builder = DataType.GraphDataType.newBuilder(); builder.setElementOpt( protoElementOpt(((GraphSchemaType) dataType).getScanOpt())); - if (dataType instanceof GraphSchemaTypeList) { - ((GraphSchemaTypeList) dataType) - .forEach( - k -> { - builder.addGraphDataType( - protoElementType(k, isColumnId)); - }); - } else { - builder.addGraphDataType( - protoElementType((GraphSchemaType) dataType, isColumnId)); - } + ((GraphSchemaType) dataType) + .getSchemaTypes() + .forEach( + k -> builder.addGraphDataType(protoElementType(k, isColumnId))); return DataType.IrDataType.newBuilder().setGraphType(builder.build()).build(); } throw new UnsupportedOperationException( @@ -322,14 +315,17 @@ public static final DataType.GraphDataType.GraphElementType protoElementType( public static final DataType.GraphDataType.GraphElementLabel protoElementLabel( GraphLabelType labelType) { + Preconditions.checkArgument( + labelType.getLabelsEntry().size() == 1, + "can not convert label=" + labelType + " to proto 'GraphElementLabel'"); + GraphLabelType.Entry entry = labelType.getSingleLabelEntry(); DataType.GraphDataType.GraphElementLabel.Builder builder = - DataType.GraphDataType.GraphElementLabel.newBuilder() - .setLabel(labelType.getLabelId()); - if (labelType.getSrcLabelId() != null) { - builder.setSrcLabel(Int32Value.of(labelType.getSrcLabelId())); + DataType.GraphDataType.GraphElementLabel.newBuilder().setLabel(entry.getLabelId()); + if (entry.getSrcLabelId() != null) { + builder.setSrcLabel(Int32Value.of(entry.getSrcLabelId())); } - if (labelType.getDstLabelId() != null) { - builder.setDstLabel(Int32Value.of(labelType.getDstLabelId())); + if (entry.getDstLabelId() != null) { + builder.setDstLabel(Int32Value.of(entry.getDstLabelId())); } return builder.build(); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java index 33eb338b677c..177f83d17b61 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/AliasInference.java @@ -176,7 +176,9 @@ public static final Set getUniqueAliasList(@Nullable RelNode input, bool while (!inputsQueue.isEmpty()) { RelNode cur = inputsQueue.remove(0); for (RelDataTypeField field : cur.getRowType().getFieldList()) { - uniqueNames.add(field.getName()); + if (field.getName() != null && field.getName() != DEFAULT_NAME) { + uniqueNames.add(field.getName()); + } } if (removeAlias(cur)) { break; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index 7cb79604a7a4..9bbf8a4d6e41 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -16,8 +16,6 @@ package com.alibaba.graphscope.common.ir.tools; -import static java.util.Objects.requireNonNull; - import com.alibaba.graphscope.common.ir.meta.schema.GraphOptSchema; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate; @@ -32,19 +30,13 @@ import com.alibaba.graphscope.common.ir.rel.type.group.GraphGroupKeys; import com.alibaba.graphscope.common.ir.rel.type.order.GraphFieldCollation; import com.alibaba.graphscope.common.ir.rel.type.order.GraphRelCollations; -import com.alibaba.graphscope.common.ir.rex.*; import com.alibaba.graphscope.common.ir.rex.RexCallBinding; +import com.alibaba.graphscope.common.ir.rex.*; import com.alibaba.graphscope.common.ir.tools.config.*; -import com.alibaba.graphscope.common.ir.type.GraphNameOrId; -import com.alibaba.graphscope.common.ir.type.GraphPathType; -import com.alibaba.graphscope.common.ir.type.GraphProperty; -import com.alibaba.graphscope.common.ir.type.GraphSchemaType; +import com.alibaba.graphscope.common.ir.type.*; import com.alibaba.graphscope.gremlin.Utils; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - +import com.google.common.base.Preconditions; +import com.google.common.collect.*; import org.apache.calcite.plan.*; import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.RelFieldCollation; @@ -60,14 +52,19 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Litmus; +import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Pair; +import org.apache.calcite.util.Sarg; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; import java.math.BigDecimal; import java.util.*; +import java.util.regex.Pattern; import java.util.stream.Collectors; +import static java.util.Objects.requireNonNull; + /** * Integrate interfaces to build algebra structures, * including {@link RexNode} for expressions and {@link RelNode} for operators @@ -418,12 +415,13 @@ public RexGraphVariable variable(@Nullable String alias, String property) { + "]"); } if (property.equals(GraphProperty.LABEL_KEY)) { + GraphSchemaType schemaType = (GraphSchemaType) aliasField.getType(); return RexGraphVariable.of( aliasField.getIndex(), new GraphProperty(GraphProperty.Opt.LABEL), columnField.left, varName, - getTypeFactory().createSqlType(SqlTypeName.CHAR)); + schemaType.getLabelType()); } else if (property.equals(GraphProperty.ID_KEY)) { return RexGraphVariable.of( aliasField.getIndex(), @@ -580,7 +578,14 @@ private RexNode call_(SqlOperator operator, List operandList) { // derive return type RelDataType returnType = operator.inferReturnType(callBinding); // derive unknown types of operands - operandList = inferOperandTypes(operator, returnType, operandList); + operandList = + inferOperandTypes(operator, returnType, convertOperands(operator, operandList)); + final RexBuilder builder = cluster.getRexBuilder(); + return builder.makeCall(returnType, operator, operandList); + } + + // convert operands of the operator in some special cases + private List convertOperands(SqlOperator operator, List operandList) { if (operator.getKind() == SqlKind.EXTRACT) { RexNode intervalOperand = operandList.get(0); if (intervalOperand instanceof RexLiteral @@ -592,11 +597,11 @@ private RexNode call_(SqlOperator operator, List operandList) { getRexBuilder() .makeFlag(intervalType.getIntervalQualifier().getStartUnit())); newOperands.add(operandList.get(1)); - operandList = newOperands; + return newOperands; } } - final RexBuilder builder = cluster.getRexBuilder(); - return builder.makeCall(returnType, operator, operandList); + // todo: convert label names to ids + return operandList; } private List inferOperandTypes( @@ -611,10 +616,16 @@ private List inferOperandTypes( List typeInferredOperands = new ArrayList<>(operandList.size()); GraphRexBuilder rexBuilder = (GraphRexBuilder) this.getRexBuilder(); for (int i = 0; i < operandList.size(); ++i) { - typeInferredOperands.add( - operandList - .get(i) - .accept(new RexNodeTypeRefresher(newTypes[i], rexBuilder))); + RexNode rexNode = operandList.get(i); + if (rexNode instanceof RexGraphDynamicParam) { + RexGraphDynamicParam graphDynamicParam = (RexGraphDynamicParam) rexNode; + rexNode = + rexBuilder.makeGraphDynamicParam( + newTypes[i], + graphDynamicParam.getName(), + graphDynamicParam.getIndex()); + } + typeInferredOperands.add(rexNode); } return typeInferredOperands; } else { @@ -637,7 +648,8 @@ private boolean isCurrentSupported(SqlOperator operator) { || sqlKind == SqlKind.ARRAY_VALUE_CONSTRUCTOR || sqlKind == SqlKind.IS_NULL || sqlKind == SqlKind.IS_NOT_NULL - || sqlKind == SqlKind.EXTRACT; + || sqlKind == SqlKind.EXTRACT + || sqlKind == SqlKind.SEARCH; } @Override @@ -679,23 +691,190 @@ public GraphBuilder filter(Iterable conditions) { AliasInference.SIMPLE_NAME(AliasInference.DEFAULT_NAME), AliasInference.DEFAULT_ID)); // add condition into table scan - if (ObjectUtils.isEmpty(tableScan.getFilters())) { - tableScan.setFilters(ImmutableList.of(condition)); - } else { - ImmutableList.Builder listBuilder = new ImmutableList.Builder(); - listBuilder.addAll(tableScan.getFilters()).add(condition); - tableScan.setFilters( - ImmutableList.of( - RexUtil.composeConjunction( - this.getRexBuilder(), listBuilder.build()))); - } // pop the filter from the inner stack - replaceTop(tableScan); + replaceTop(fuseFilters(tableScan, condition)); } } return builder; } + private AbstractBindableTableScan fuseFilters( + AbstractBindableTableScan tableScan, RexNode condition) { + ImmutableList originalFilters = tableScan.getFilters(); + List labelValues = Lists.newArrayList(); + List uniqueKeyFilters = Lists.newArrayList(); + List extraFilters = Lists.newArrayList(); + classifyFilters(tableScan, condition, labelValues, uniqueKeyFilters, extraFilters); + if (!labelValues.isEmpty()) { + GraphLabelType labelType = + ((GraphSchemaType) tableScan.getRowType().getFieldList().get(0).getType()) + .getLabelType(); + List labelsToKeep = + labelType.getLabelsEntry().stream() + .filter(k -> labelValues.contains(k.getLabel())) + .map(k -> k.getLabel()) + .collect(Collectors.toList()); + Preconditions.checkArgument( + !labelsToKeep.isEmpty(), + "cannot find common labels between values= " + labelValues + " and label=", + labelType); + if (labelsToKeep.size() < labelType.getLabelsEntry().size()) { + GraphBuilder builder = + (GraphBuilder) + GraphPlanner.relBuilderFactory.create( + getCluster(), getRelOptSchema()); + LabelConfig newLabelConfig = new LabelConfig(false); + labelsToKeep.forEach(k -> newLabelConfig.addLabel(k)); + if (tableScan instanceof GraphLogicalSource) { + builder.source( + new SourceConfig( + ((GraphLogicalSource) tableScan).getOpt(), + newLabelConfig, + tableScan.getAliasName())); + } else if (tableScan instanceof GraphLogicalExpand) { + ((GraphBuilder) builder.push(tableScan.getInput(0))) + .expand( + new ExpandConfig( + ((GraphLogicalExpand) tableScan).getOpt(), + newLabelConfig, + tableScan.getAliasName())); + } else if (tableScan instanceof GraphLogicalGetV) { + ((GraphBuilder) builder.push(tableScan.getInput(0))) + .getV( + new GetVConfig( + ((GraphLogicalGetV) tableScan).getOpt(), + newLabelConfig, + tableScan.getAliasName())); + } + if (builder.size() > 0) { + // check if the property still exist after updating the label type + RexVisitor propertyChecker = + new RexVisitorImpl(true) { + @Override + public Void visitInputRef(RexInputRef inputRef) { + if (inputRef instanceof RexGraphVariable) { + RexGraphVariable variable = (RexGraphVariable) inputRef; + String[] splits = + variable.getName() + .split( + Pattern.quote( + AliasInference.DELIMITER)); + if (splits.length > 1) { + builder.variable(null, splits[1]); + } + } + return null; + } + }; + if (!ObjectUtils.isEmpty(originalFilters)) { + originalFilters.forEach(k -> ((RexNode) k).accept(propertyChecker)); + builder.filter(originalFilters); + } + if (!extraFilters.isEmpty()) { + extraFilters.forEach(k -> k.accept(propertyChecker)); + } + tableScan = (AbstractBindableTableScan) builder.build(); + } + } + } + if (tableScan instanceof GraphLogicalSource && !uniqueKeyFilters.isEmpty()) { + GraphLogicalSource source = (GraphLogicalSource) tableScan; + if (source.getUniqueKeyFilters() != null) { + uniqueKeyFilters.add(0, source.getUniqueKeyFilters()); + } + source.setUniqueKeyFilters( + RexUtil.composeConjunction(this.getRexBuilder(), uniqueKeyFilters)); + } + if (!extraFilters.isEmpty()) { + if (ObjectUtils.isEmpty(originalFilters)) { + tableScan.setFilters( + ImmutableList.of( + RexUtil.composeConjunction(this.getRexBuilder(), extraFilters))); + } else { + List combineFilters = Lists.newArrayList(originalFilters); + combineFilters.addAll(extraFilters); + tableScan.setFilters( + ImmutableList.of( + RexUtil.composeConjunction(this.getRexBuilder(), combineFilters))); + } + } + return tableScan; + } + + private void classifyFilters( + AbstractBindableTableScan tableScan, + RexNode condition, + List labelValues, + List uniqueKeyFilters, + List filters) { + List conjunctions = RelOptUtil.conjunctions(condition); + List filtersToRemove = Lists.newArrayList(); + for (RexNode conjunction : conjunctions) { + if (conjunction instanceof RexCall) { + RexCall rexCall = (RexCall) conjunction; + if (rexCall.getOperator().getKind() == SqlKind.EQUALS + || rexCall.getOperator().getKind() == SqlKind.SEARCH) { + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (left.getType() instanceof GraphLabelType && right instanceof RexLiteral) { + filtersToRemove.add(condition); + labelValues.addAll( + getValuesAsList(((RexLiteral) right).getValueAs(Comparable.class))); + break; + } else if (left instanceof RexLiteral + && right.getType() instanceof GraphLabelType) { + filtersToRemove.add(condition); + labelValues.addAll( + getValuesAsList(((RexLiteral) left).getValueAs(Comparable.class))); + break; + } + if (tableScan instanceof GraphLogicalSource) { + if (isUniqueKey(left) && right instanceof RexLiteral) { + filtersToRemove.add(condition); + uniqueKeyFilters.add(condition); + break; + } else if (left instanceof RexLiteral && isUniqueKey(right)) { + filtersToRemove.add(condition); + uniqueKeyFilters.add(condition); + break; + } + } + } + } + } + if (!filtersToRemove.isEmpty()) { + conjunctions.removeAll(filtersToRemove); + } + filters.addAll(conjunctions); + } + + private boolean isUniqueKey(RexNode rexNode) { + if (rexNode instanceof RexGraphVariable) { + RexGraphVariable variable = (RexGraphVariable) rexNode; + return variable.getProperty() != null + && variable.getProperty().getOpt() == GraphProperty.Opt.ID; + } + return false; + } + + private List getValuesAsList(Comparable value) { + ImmutableList.Builder labelBuilder = ImmutableList.builder(); + if (value instanceof NlsString) { + labelBuilder.add(((NlsString) value).getValue()); + } else if (value instanceof Sarg) { + Sarg sarg = (Sarg) value; + if (sarg.isPoints()) { + Set> rangeSets = sarg.rangeSet.asRanges(); + for (Range range : rangeSets) { + labelBuilder.addAll(getValuesAsList(range.lowerEndpoint())); + } + } + } else { + labelBuilder.add(value); + } + return labelBuilder.build(); + } + // return the top node if its type is Filter, otherwise null private Filter topFilter() { if (this.size() > 0 && this.peek() instanceof Filter) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 80e85957ee58..4a2ef40d8821 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -72,7 +72,7 @@ public class GraphPlanner { private final RelOptPlanner optPlanner; private final RexBuilder rexBuilder; private final AtomicLong idGenerator; - private static final RelBuilderFactory relBuilderFactory = + public static final RelBuilderFactory relBuilderFactory = (RelOptCluster cluster, @Nullable RelOptSchema schema) -> GraphBuilder.create(null, (GraphOptCluster) cluster, schema); @@ -258,6 +258,7 @@ public static void main(String[] args) throws Exception { // write physical plan to file try (PhysicalBuilder physicalBuilder = summary.getPhysicalBuilder()) { FileUtils.writeByteArrayToFile(new File(args[2]), physicalBuilder.build()); + physicalBuilder.explain(); } // write stored procedure meta to file LogicalPlan logicalPlan = summary.getLogicalPlan(); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphLabelType.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphLabelType.java index aab24c674cd6..2c022e6b669b 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphLabelType.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphLabelType.java @@ -16,81 +16,165 @@ package com.alibaba.graphscope.common.ir.type; +import com.google.common.collect.ImmutableList; + +import org.apache.calcite.sql.type.AbstractSqlType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.checkerframework.checker.nullness.qual.Nullable; +import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * Maintain label for each Entity or Relation: Entity(label), Relation(Label, srcLabel, dstLabel). */ -public class GraphLabelType { - public static GraphLabelType DEFAULT = new GraphLabelType(); - - private String label; - private Integer labelId; - @Nullable private String srcLabel; - @Nullable private Integer srcLabelId; - @Nullable private String dstLabel; - @Nullable private Integer dstLabelId; - - public GraphLabelType() { - this.label = StringUtils.EMPTY; - this.labelId = -1; - } - - public GraphLabelType label(String label) { - Objects.requireNonNull(label); - this.label = label; - return this; - } - - public GraphLabelType labelId(int labelId) { - this.labelId = labelId; - return this; - } +public class GraphLabelType extends AbstractSqlType { + private final List labels; - public GraphLabelType srcLabel(String srcLabel) { - this.srcLabel = srcLabel; - return this; + public GraphLabelType(Entry label) { + this(ImmutableList.of(label)); } - public GraphLabelType srcLabelId(int srcLabelId) { - this.srcLabelId = srcLabelId; - return this; + public GraphLabelType(List labels) { + this(labels, SqlTypeName.CHAR); } - public GraphLabelType dstLabel(String dstLabel) { - this.dstLabel = dstLabel; - return this; + public GraphLabelType(Entry label, SqlTypeName typeName) { + this(ImmutableList.of(label), typeName); } - public GraphLabelType dstLabelId(int dstLabelId) { - this.dstLabelId = dstLabelId; - return this; + public GraphLabelType(List labels, SqlTypeName typeName) { + super(typeName, false, null); + this.labels = ObjectUtils.requireNonEmpty(labels); + this.computeDigest(); } - public String getLabel() { - return label; + public Entry getSingleLabelEntry() { + return labels.get(0); } - public Integer getLabelId() { - return labelId; + public List getLabelsEntry() { + return Collections.unmodifiableList(labels); } - public @Nullable String getSrcLabel() { - return srcLabel; + public List getLabelsString() { + return getLabelsEntry().stream() + .map(k -> k.toString()) + .collect(Collectors.toUnmodifiableList()); } - public @Nullable Integer getSrcLabelId() { - return srcLabelId; + @Override + protected void generateTypeString(StringBuilder stringBuilder, boolean b) { + stringBuilder.append(getLabelsString()); } - public @Nullable String getDstLabel() { - return dstLabel; + public void removeLabels(List labelsToRemove) { + this.labels.removeAll(labelsToRemove); } - public @Nullable Integer getDstLabelId() { - return dstLabelId; + public static class Entry { + private String label; + private Integer labelId; + @Nullable private String srcLabel; + @Nullable private Integer srcLabelId; + @Nullable private String dstLabel; + @Nullable private Integer dstLabelId; + + public Entry() { + this.label = StringUtils.EMPTY; + this.labelId = -1; + } + + public Entry label(String label) { + Objects.requireNonNull(label); + this.label = label; + return this; + } + + public Entry labelId(int labelId) { + this.labelId = labelId; + return this; + } + + public Entry srcLabel(String srcLabel) { + this.srcLabel = srcLabel; + return this; + } + + public Entry srcLabelId(int srcLabelId) { + this.srcLabelId = srcLabelId; + return this; + } + + public Entry dstLabel(String dstLabel) { + this.dstLabel = dstLabel; + return this; + } + + public Entry dstLabelId(int dstLabelId) { + this.dstLabelId = dstLabelId; + return this; + } + + public String getLabel() { + return label; + } + + public Integer getLabelId() { + return labelId; + } + + public @Nullable String getSrcLabel() { + return srcLabel; + } + + public @Nullable Integer getSrcLabelId() { + return srcLabelId; + } + + public @Nullable String getDstLabel() { + return dstLabel; + } + + public @Nullable Integer getDstLabelId() { + return dstLabelId; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + if (srcLabel == null || dstLabel == null) { + builder.append("VertexLabel("); + builder.append(label); + builder.append(")"); + } else { + builder.append("EdgeLabel("); + builder.append(label + ", " + srcLabel + ", " + dstLabel); + builder.append(")"); + } + return builder.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Entry entry = (Entry) o; + return Objects.equals(label, entry.label) + && Objects.equals(labelId, entry.labelId) + && Objects.equals(srcLabel, entry.srcLabel) + && Objects.equals(srcLabelId, entry.srcLabelId) + && Objects.equals(dstLabel, entry.dstLabel) + && Objects.equals(dstLabelId, entry.dstLabelId); + } + + @Override + public int hashCode() { + return Objects.hash(label, labelId, srcLabel, srcLabelId, dstLabel, dstLabelId); + } } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java index a4beb93869fa..60268d6960f8 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java @@ -17,23 +17,27 @@ package com.alibaba.graphscope.common.ir.type; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import org.apache.calcite.linq4j.Ord; -import org.apache.calcite.rel.type.RelDataTypeFamily; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rel.type.RelRecordType; -import org.apache.calcite.rel.type.StructKind; +import org.apache.calcite.rel.type.*; +import org.apache.commons.lang3.ObjectUtils; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * Denote DataType of an entity or a relation, including opt, label and attributes */ public class GraphSchemaType extends RelRecordType { - protected GraphOpt.Source scanOpt; - protected GraphLabelType labelType; + private final GraphOpt.Source scanOpt; + private final GraphLabelType labelType; + private final List fuzzySchemaTypes; /** * @param scanOpt entity or relation @@ -45,11 +49,6 @@ public GraphSchemaType( this(scanOpt, labelType, fields, false); } - protected GraphSchemaType( - GraphOpt.Source scanOpt, List fields, boolean isNullable) { - this(scanOpt, GraphLabelType.DEFAULT, fields, isNullable); - } - /** * add a constructor to accept {@code isNullable}, a nullable GraphSchemaType will be created after left outer join * @param scanOpt @@ -62,11 +61,77 @@ public GraphSchemaType( GraphLabelType labelType, List fields, boolean isNullable) { + this(scanOpt, labelType, fields, ImmutableList.of(), isNullable); + } + + protected GraphSchemaType( + GraphOpt.Source scanOpt, + GraphLabelType labelType, + List fields, + List fuzzySchemaTypes, + boolean isNullable) { super(StructKind.NONE, fields, isNullable); this.scanOpt = scanOpt; + this.fuzzySchemaTypes = Objects.requireNonNull(fuzzySchemaTypes); this.labelType = labelType; } + public static GraphSchemaType create( + List list, RelDataTypeFactory typeFactory) { + return create(list, typeFactory, false); + } + + public static GraphSchemaType create( + List list, RelDataTypeFactory typeFactory, boolean isNullable) { + ObjectUtils.requireNonEmpty(list, "schema type list should not be empty"); + if (list.size() == 1) { + return list.get(0); + } + GraphOpt.Source scanOpt = list.get(0).getScanOpt(); + List labelOpts = Lists.newArrayList(); + List fields = Lists.newArrayList(); + List commonFields = Lists.newArrayList(list.get(0).getFieldList()); + List fuzzyEntries = Lists.newArrayList(); + for (GraphSchemaType type : list) { + Preconditions.checkArgument( + !type.fuzzy(), + "fuzzy label types nested in list of " + + GraphSchemaType.class + + " is considered to be invalid here"); + labelOpts.add( + "{label=" + + type.getLabelType().getLabelsString() + + ", opt=" + + type.scanOpt + + "}"); + if (type.getScanOpt() != scanOpt) { + throw new IllegalArgumentException( + "fuzzy label types should have the same opt, but is " + labelOpts); + } + fields.addAll(type.getFieldList()); + commonFields.retainAll(type.getFieldList()); + fuzzyEntries.addAll(type.getLabelType().getLabelsEntry()); + } + fields = + fields.stream() + .distinct() + .map( + k -> { + if (!commonFields.contains( + k)) { // can be optional for some labels + return new RelDataTypeFieldImpl( + k.getName(), + k.getIndex(), + typeFactory.createTypeWithNullability( + k.getType(), true)); + } + return k; + }) + .collect(Collectors.toList()); + return new GraphSchemaType( + scanOpt, new GraphLabelType(fuzzyEntries), fields, list, isNullable); + } + public GraphOpt.Source getScanOpt() { return scanOpt; } @@ -119,4 +184,14 @@ public boolean isStruct() { public RelDataTypeFamily getFamily() { return scanOpt; } + + public List getSchemaTypes() { + return ObjectUtils.isEmpty(this.fuzzySchemaTypes) + ? ImmutableList.of(this) + : Collections.unmodifiableList(this.fuzzySchemaTypes); + } + + public boolean fuzzy() { + return this.labelType.getLabelsEntry().size() > 1; + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaTypeList.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaTypeList.java deleted file mode 100644 index 7c464dd27b86..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaTypeList.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.type; - -import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; - -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.commons.lang3.ObjectUtils; - -import java.util.*; -import java.util.stream.Collectors; - -/** - * A list of {@code IrSchemaType}, to denote fuzzy conditions in a vertex or an edge, i.e. g.V() or g.V().hasLabel("person", "software") - */ -public class GraphSchemaTypeList extends GraphSchemaType implements List { - private List schemaTypes; - - public static GraphSchemaTypeList create(List list) { - return create(list, false); - } - - public static GraphSchemaTypeList create(List list, boolean isNullable) { - ObjectUtils.requireNonEmpty(list); - GraphOpt.Source scanOpt = list.get(0).getScanOpt(); - List labelOpts = new ArrayList<>(); - List fields = new ArrayList<>(); - for (GraphSchemaType type : list) { - labelOpts.add("{label=" + type.labelType.getLabel() + ", opt=" + type.scanOpt + "}"); - if (type.getScanOpt() != scanOpt) { - throw new IllegalArgumentException( - "fuzzy label types should have the same opt, but is " + labelOpts); - } - fields.addAll(type.getFieldList()); - } - return new GraphSchemaTypeList( - scanOpt, list, fields.stream().distinct().collect(Collectors.toList()), isNullable); - } - - protected GraphSchemaTypeList( - GraphOpt.Source scanOpt, - List schemaTypes, - List fields, - boolean isNullable) { - super(scanOpt, fields, isNullable); - this.schemaTypes = schemaTypes; - } - - @Override - public int size() { - return this.schemaTypes.size(); - } - - @Override - public boolean isEmpty() { - return this.schemaTypes.isEmpty(); - } - - @Override - public boolean contains(Object o) { - return this.schemaTypes.contains(o); - } - - @Override - public Iterator iterator() { - return this.schemaTypes.iterator(); - } - - @Override - public Object[] toArray() { - return this.schemaTypes.toArray(); - } - - @Override - public T[] toArray(T[] a) { - return this.schemaTypes.toArray(a); - } - - @Override - public boolean add(GraphSchemaType graphSchemaType) { - return this.schemaTypes.add(graphSchemaType); - } - - @Override - public boolean remove(Object o) { - return this.schemaTypes.remove(o); - } - - @Override - public boolean containsAll(Collection c) { - return this.schemaTypes.containsAll(c); - } - - @Override - public boolean addAll(Collection c) { - return this.schemaTypes.addAll(c); - } - - @Override - public boolean addAll(int index, Collection c) { - return this.schemaTypes.addAll(index, c); - } - - @Override - public boolean removeAll(Collection c) { - return this.schemaTypes.removeAll(c); - } - - @Override - public boolean retainAll(Collection c) { - return this.schemaTypes.retainAll(c); - } - - @Override - public void clear() { - this.schemaTypes.clear(); - } - - @Override - public GraphSchemaType get(int index) { - return this.schemaTypes.get(index); - } - - @Override - public GraphSchemaType set(int index, GraphSchemaType element) { - return this.schemaTypes.set(index, element); - } - - @Override - public void add(int index, GraphSchemaType element) { - this.schemaTypes.add(index, element); - } - - @Override - public GraphSchemaType remove(int index) { - return this.schemaTypes.remove(index); - } - - @Override - public int indexOf(Object o) { - return this.schemaTypes.indexOf(o); - } - - @Override - public int lastIndexOf(Object o) { - return this.schemaTypes.lastIndexOf(o); - } - - @Override - public ListIterator listIterator() { - return this.schemaTypes.listIterator(); - } - - @Override - public ListIterator listIterator(int index) { - return this.schemaTypes.listIterator(index); - } - - @Override - public List subList(int fromIndex, int toIndex) { - return this.schemaTypes.subList(fromIndex, toIndex); - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java index 4411b89ee833..4d09fbb8f4c8 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java @@ -18,7 +18,6 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; -import com.google.common.collect.Lists; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.rel.type.RelDataType; @@ -36,19 +35,18 @@ public GraphTypeFactoryImpl(Configs configs) { @Override public RelDataType createTypeWithNullability(RelDataType type, boolean nullable) { RelDataType newType; - if (type instanceof GraphSchemaTypeList) { - GraphSchemaTypeList schemaTypeList = (GraphSchemaTypeList) type; - newType = - GraphSchemaTypeList.create( - Lists.newArrayList(schemaTypeList.listIterator()), nullable); - } else if (type instanceof GraphSchemaType) { + if (type instanceof GraphSchemaType) { GraphSchemaType schemaType = (GraphSchemaType) type; - newType = - new GraphSchemaType( - schemaType.getScanOpt(), - schemaType.getLabelType(), - schemaType.getFieldList(), - nullable); + if (schemaType.getSchemaTypes().size() > 1) { + newType = GraphSchemaType.create(schemaType.getSchemaTypes(), this, nullable); + } else { + newType = + new GraphSchemaType( + schemaType.getScanOpt(), + schemaType.getLabelType(), + schemaType.getFieldList(), + nullable); + } } else { newType = super.createTypeWithNullability(type, nullable); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java index 0e0389970aa3..921fa7b3750d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java @@ -19,7 +19,6 @@ import com.alibaba.graphscope.common.ir.type.GraphLabelType; import com.alibaba.graphscope.common.ir.type.GraphPathType; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; -import com.alibaba.graphscope.common.ir.type.GraphSchemaTypeList; import com.alibaba.graphscope.common.result.RecordParser; import com.alibaba.graphscope.gaia.proto.Common; import com.alibaba.graphscope.gaia.proto.IrResult; @@ -224,18 +223,20 @@ protected AnyValue parseValue(Common.Value value, @Nullable RelDataType dataType } } - private String getLabelName(Common.NameOrId nameOrId, List labelTypes) { + private String getLabelName(Common.NameOrId nameOrId, @Nullable GraphLabelType labelTypes) { switch (nameOrId.getItemCase()) { case NAME: return nameOrId.getName(); case ID: default: List labelIds = new ArrayList<>(); - for (GraphLabelType labelType : labelTypes) { - if (labelType.getLabelId() == nameOrId.getId()) { - return labelType.getLabel(); + if (labelTypes != null) { + for (GraphLabelType.Entry labelType : labelTypes.getLabelsEntry()) { + if (labelType.getLabelId() == nameOrId.getId()) { + return labelType.getLabel(); + } + labelIds.add(labelType.getLabelId()); } - labelIds.add(labelType.getLabelId()); } logger.warn( "label id={} not found, expected ids are {}", nameOrId.getId(), labelIds); @@ -243,16 +244,18 @@ private String getLabelName(Common.NameOrId nameOrId, List label } } - private String getSrcLabelName(Common.NameOrId nameOrId, List labelTypes) { + private String getSrcLabelName(Common.NameOrId nameOrId, @Nullable GraphLabelType labelTypes) { switch (nameOrId.getItemCase()) { case NAME: return nameOrId.getName(); case ID: default: List labelIds = new ArrayList<>(); - for (GraphLabelType labelType : labelTypes) { - if (labelType.getSrcLabelId() == nameOrId.getId()) { - return labelType.getSrcLabel(); + if (labelTypes != null) { + for (GraphLabelType.Entry labelType : labelTypes.getLabelsEntry()) { + if (labelType.getSrcLabelId() == nameOrId.getId()) { + return labelType.getSrcLabel(); + } } } logger.warn( @@ -263,16 +266,18 @@ private String getSrcLabelName(Common.NameOrId nameOrId, List la } } - private String getDstLabelName(Common.NameOrId nameOrId, List labelTypes) { + private String getDstLabelName(Common.NameOrId nameOrId, @Nullable GraphLabelType labelTypes) { switch (nameOrId.getItemCase()) { case NAME: return nameOrId.getName(); case ID: default: List labelIds = new ArrayList<>(); - for (GraphLabelType labelType : labelTypes) { - if (labelType.getDstLabelId() == nameOrId.getId()) { - return labelType.getDstLabel(); + if (labelTypes != null) { + for (GraphLabelType.Entry labelType : labelTypes.getLabelsEntry()) { + if (labelType.getDstLabelId() == nameOrId.getId()) { + return labelType.getDstLabel(); + } } } logger.warn( @@ -283,18 +288,12 @@ private String getDstLabelName(Common.NameOrId nameOrId, List la } } - private List getLabelTypes(RelDataType dataType) { - List labelTypes = Lists.newArrayList(); - if (dataType instanceof GraphSchemaTypeList) { - ((GraphSchemaTypeList) dataType) - .forEach( - k -> { - labelTypes.add(k.getLabelType()); - }); - } else if (dataType instanceof GraphSchemaType) { - labelTypes.add(((GraphSchemaType) dataType).getLabelType()); + private @Nullable GraphLabelType getLabelTypes(RelDataType dataType) { + if (dataType instanceof GraphSchemaType) { + return ((GraphSchemaType) dataType).getLabelType(); + } else { + return null; } - return labelTypes; } private RelDataType getVertexType(RelDataType graphPathType) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/parser/GremlinAntlr4Parser.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/parser/GremlinAntlr4Parser.java new file mode 100644 index 000000000000..24c8c718427a --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/parser/GremlinAntlr4Parser.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4x.parser; + +import com.alibaba.graphscope.common.antlr4.Antlr4Parser; +import com.alibaba.graphscope.common.antlr4.SyntaxErrorListener; +import com.alibaba.graphscope.grammar.GremlinGSLexer; +import com.alibaba.graphscope.grammar.GremlinGSParser; + +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * parse gremlin DSL to antlr tree + */ +public class GremlinAntlr4Parser implements Antlr4Parser { + @Override + public ParseTree parse(String statement) { + GremlinGSLexer lexer = new GremlinGSLexer(CharStreams.fromString(statement)); + // reset error listeners on lexer + lexer.removeErrorListeners(); + lexer.addErrorListener(new SyntaxErrorListener()); + final GremlinGSParser parser = new GremlinGSParser(new CommonTokenStream(lexer)); + // setup error handler on parser + parser.setErrorHandler(new DefaultErrorStrategy()); + // reset error listeners on parser + parser.removeErrorListeners(); + parser.addErrorListener(new SyntaxErrorListener()); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + return parser.query(); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java new file mode 100644 index 000000000000..64b1f64719b6 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java @@ -0,0 +1,254 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4x.visitor; + +import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; +import com.alibaba.graphscope.grammar.GremlinGSBaseVisitor; +import com.alibaba.graphscope.grammar.GremlinGSParser; +import com.alibaba.graphscope.gremlin.antlr4.GenericLiteralVisitor; +import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; + +import org.apache.calcite.rex.RexNode; + +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + +public class ExpressionVisitor extends GremlinGSBaseVisitor { + private final RexNode propertyKey; + private final GraphBuilder builder; + + public ExpressionVisitor(GraphBuilder builder, RexNode propertyKey) { + this.builder = Objects.requireNonNull(builder); + this.propertyKey = Objects.requireNonNull(propertyKey); + } + + @Override + public RexNode visitTraversalPredicate(GremlinGSParser.TraversalPredicateContext ctx) { + switch (ctx.getChildCount()) { + case 1: + // handle simple predicate + return visitChildren(ctx); + case 6: + final int childIndexOfParameterOperator = 2; + final int childIndexOfCaller = 0; + final int childIndexOfArgument = 4; + + if (ctx.getChild(childIndexOfParameterOperator).getText().equals("or")) { + // handle or + return builder.call( + GraphStdOperatorTable.OR, + visit(ctx.getChild(childIndexOfCaller)), + visit(ctx.getChild(childIndexOfArgument))); + } else { + // handle and + return builder.call( + GraphStdOperatorTable.AND, + visit(ctx.getChild(childIndexOfCaller)), + visit(ctx.getChild(childIndexOfArgument))); + } + default: + throw new UnsupportedEvalException( + ctx.getClass(), + "unexpected number of children in TraversalPredicateContext " + + ctx.getChildCount()); + } + } + + @Override + public RexNode visitTraversalPredicate_eq(GremlinGSParser.TraversalPredicate_eqContext ctx) { + return builder.call( + GraphStdOperatorTable.EQUALS, + propertyKey, + builder.literal( + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral()))); + } + + @Override + public RexNode visitTraversalPredicate_neq(GremlinGSParser.TraversalPredicate_neqContext ctx) { + return builder.call( + GraphStdOperatorTable.NOT_EQUALS, + propertyKey, + builder.literal( + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral()))); + } + + @Override + public RexNode visitTraversalPredicate_lt(GremlinGSParser.TraversalPredicate_ltContext ctx) { + return builder.call( + GraphStdOperatorTable.LESS_THAN, + propertyKey, + builder.literal( + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral()))); + } + + @Override + public RexNode visitTraversalPredicate_lte(GremlinGSParser.TraversalPredicate_lteContext ctx) { + return builder.call( + GraphStdOperatorTable.LESS_THAN_OR_EQUAL, + propertyKey, + builder.literal( + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral()))); + } + + @Override + public RexNode visitTraversalPredicate_gt(GremlinGSParser.TraversalPredicate_gtContext ctx) { + return builder.call( + GraphStdOperatorTable.GREATER_THAN, + propertyKey, + builder.literal( + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral()))); + } + + @Override + public RexNode visitTraversalPredicate_gte(GremlinGSParser.TraversalPredicate_gteContext ctx) { + return builder.call( + GraphStdOperatorTable.GREATER_THAN_OR_EQUAL, + propertyKey, + builder.literal( + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral()))); + } + + @Override + public RexNode visitTraversalPredicate_within( + GremlinGSParser.TraversalPredicate_withinContext ctx) { + Object[] points = GenericLiteralVisitor.getGenericLiteralList(ctx.genericLiteralList()); + return builder.getRexBuilder() + .makeIn( + propertyKey, + Arrays.asList(points).stream() + .map(k -> builder.literal(k)) + .collect(Collectors.toList())); + } + + @Override + public RexNode visitTraversalPredicate_without( + GremlinGSParser.TraversalPredicate_withoutContext ctx) { + Object[] points = GenericLiteralVisitor.getGenericLiteralList(ctx.genericLiteralList()); + return builder.not( + builder.getRexBuilder() + .makeIn( + propertyKey, + Arrays.asList(points).stream() + .map(k -> builder.literal(k)) + .collect(Collectors.toList()))); + } + + @Override + public RexNode visitTraversalPredicate_not(GremlinGSParser.TraversalPredicate_notContext ctx) { + return builder.not(visitChildren(ctx)); + } + + @Override + public RexNode visitTraversalPredicate_inside( + GremlinGSParser.TraversalPredicate_insideContext ctx) { + Number lower = + (Number) + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral(0)); + Number upper = + (Number) + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral(1)); + return builder.getRexBuilder() + .makeBetween( + propertyKey, + builder.literal(new BigDecimal(lower.longValue() + 1)), + builder.literal(new BigDecimal(upper.longValue() - 1))); + } + + @Override + public RexNode visitTraversalPredicate_outside( + GremlinGSParser.TraversalPredicate_outsideContext ctx) { + Number lower = + (Number) + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral(0)); + Number upper = + (Number) + GenericLiteralVisitor.getInstance() + .visitGenericLiteral(ctx.genericLiteral(1)); + return builder.not( + builder.getRexBuilder() + .makeBetween( + propertyKey, + builder.literal(new BigDecimal(lower.longValue() + 1)), + builder.literal(new BigDecimal(upper.longValue() - 1)))); + } + + @Override + public RexNode visitTraversalPredicate_startingWith( + GremlinGSParser.TraversalPredicate_startingWithContext ctx) { + String posixRegex = "^" + ctx.getText(); + return builder.call( + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, builder.literal(posixRegex)); + } + + @Override + public RexNode visitTraversalPredicate_notStartingWith( + GremlinGSParser.TraversalPredicate_notStartingWithContext ctx) { + String posixRegex = "^" + ctx.getText(); + return builder.not( + builder.call( + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + builder.literal(posixRegex))); + } + + @Override + public RexNode visitTraversalPredicate_endingWith( + GremlinGSParser.TraversalPredicate_endingWithContext ctx) { + String posixRegex = ctx.getText() + "$"; + return builder.call( + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, builder.literal(posixRegex)); + } + + @Override + public RexNode visitTraversalPredicate_notEndingWith( + GremlinGSParser.TraversalPredicate_notEndingWithContext ctx) { + String posixRegex = ctx.getText() + "$"; + return builder.not( + builder.call( + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + builder.literal(posixRegex))); + } + + @Override + public RexNode visitTraversalPredicate_containing( + GremlinGSParser.TraversalPredicate_containingContext ctx) { + String posixRegex = ".*" + ctx.getText() + "*."; + return builder.call( + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, builder.literal(posixRegex)); + } + + @Override + public RexNode visitTraversalPredicate_notContaining( + GremlinGSParser.TraversalPredicate_notContainingContext ctx) { + String posixRegex = ".*" + ctx.getText() + "*."; + return builder.not( + builder.call( + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + builder.literal(posixRegex))); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java new file mode 100644 index 000000000000..c8d0d18893f3 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java @@ -0,0 +1,183 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4x.visitor; + +import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; +import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.alibaba.graphscope.common.ir.tools.config.SourceConfig; +import com.alibaba.graphscope.common.ir.type.GraphProperty; +import com.alibaba.graphscope.grammar.GremlinGSBaseVisitor; +import com.alibaba.graphscope.grammar.GremlinGSParser; +import com.alibaba.graphscope.gremlin.antlr4.GenericLiteralVisitor; +import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; +import com.google.common.collect.Lists; + +import org.apache.calcite.rex.RexNode; +import org.apache.commons.lang3.ObjectUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class GraphBuilderVisitor extends GremlinGSBaseVisitor { + private final GraphBuilder builder; + + public GraphBuilderVisitor(GraphBuilder builder) { + this.builder = builder; + } + + @Override + public GraphBuilder visitTraversalSourceSpawnMethod_V( + GremlinGSParser.TraversalSourceSpawnMethod_VContext ctx) { + return builder.source(new SourceConfig(GraphOpt.Source.VERTEX)); + } + + @Override + public GraphBuilder visitTraversalSourceSpawnMethod_E( + GremlinGSParser.TraversalSourceSpawnMethod_EContext ctx) { + return builder.source(new SourceConfig(GraphOpt.Source.EDGE)); + } + + @Override + public GraphBuilder visitTraversalMethod_hasLabel( + GremlinGSParser.TraversalMethod_hasLabelContext ctx) { + if (ctx.stringLiteral() != null) { + String label = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()); + if (ObjectUtils.isEmpty(ctx.stringLiteralList())) { + return builder.filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal(label))); + } else { + List labelNodes = Lists.newArrayList(builder.literal(label)); + String[] otherLabels = + GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + for (String other : otherLabels) { + labelNodes.add(builder.literal(other)); + } + RexNode labelFilters = + builder.getRexBuilder() + .makeIn( + builder.variable(null, GraphProperty.LABEL_KEY), + labelNodes); + return builder.filter(labelFilters); + } + } + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [hasLabel('str')] or hasLabel('str1', ...)"); + } + + @Override + public GraphBuilder visitTraversalMethod_hasId( + GremlinGSParser.TraversalMethod_hasIdContext ctx) { + Object[] ids = + GenericLiteralVisitor.getIntegerLiteralExpr( + ctx.nonEmptyIntegerLiteralList().integerLiteralExpr()); + if (ids.length == 1) { + return builder.filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.ID_KEY), + builder.literal(ids[0]))); + } else if (ids.length > 1) { + List idNodes = + Arrays.asList(ids).stream() + .map(k -> builder.literal(k)) + .collect(Collectors.toList()); + RexNode idFilters = + builder.getRexBuilder() + .makeIn(builder.variable(null, GraphProperty.ID_KEY), idNodes); + return builder.filter(idFilters); + } + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [hasId(1)] or hasId(1, 2, ...)"); + } + + @Override + public GraphBuilder visitTraversalMethod_has(GremlinGSParser.TraversalMethod_hasContext ctx) { + String notice = + "supported pattern is [has('key', 'value')] or [has('key', P)] or [has('label'," + + " 'key', 'value')] or [has('label', 'key', P)]"; + int childCount = ctx.getChildCount(); + if (childCount == 6 && ctx.genericLiteral() != null) { // g.V().has("name", "marko") + String propertyKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)); + Object propertyValue = + GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral()); + return builder.filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, propertyKey), + builder.literal(propertyValue))); + } else if (childCount == 6 + && ctx.traversalPredicate() != null) { // g.V().has("name", P.eq("marko")) + String propertyKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)); + ExpressionVisitor exprVisitor = + new ExpressionVisitor(this.builder, builder.variable(null, propertyKey)); + return builder.filter(exprVisitor.visitTraversalPredicate(ctx.traversalPredicate())); + } else if (childCount == 8 + && ctx.genericLiteral() != null) { // g.V().has("person", "name", "marko") + String labelValue = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)); + String propertyKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(1)); + Object propertyValue = + GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral()); + return builder.filter( + builder.call( + GraphStdOperatorTable.AND, + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal(labelValue)), + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, propertyKey), + builder.literal(propertyValue)))); + } else if (childCount == 8 + && ctx.traversalPredicate() != null) { // g.V().has("person", "name", P.eq("marko")) + String labelValue = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)); + String propertyKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(1)); + ExpressionVisitor exprVisitor = + new ExpressionVisitor(this.builder, builder.variable(null, propertyKey)); + return builder.filter( + builder.call( + GraphStdOperatorTable.AND, + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.LABEL_KEY), + builder.literal(labelValue)), + exprVisitor.visitTraversalPredicate(ctx.traversalPredicate()))); + } else if (childCount == 4 && ctx.stringLiteral() != null) { // g.V().has("name") + String propertyKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)); + return builder.filter( + builder.call( + GraphStdOperatorTable.IS_NOT_NULL, + builder.variable(null, propertyKey))); + } else { + throw new UnsupportedEvalException(ctx.getClass(), notice); + } + } + + @Override + public GraphBuilder visitTraversalMethod_hasNot( + GremlinGSParser.TraversalMethod_hasNotContext ctx) { + // g.V().hasNot("name") + String propertyKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()); + return builder.filter( + builder.call(GraphStdOperatorTable.IS_NULL, builder.variable(null, propertyKey))); + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java index e64f23570e52..96ca190cf6d8 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java @@ -18,10 +18,11 @@ import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; -import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; -import com.alibaba.graphscope.common.ir.tools.config.LabelConfig; -import com.alibaba.graphscope.common.ir.tools.config.SourceConfig; +import com.alibaba.graphscope.common.ir.tools.config.*; +import com.alibaba.graphscope.common.ir.type.GraphProperty; +import com.google.common.collect.ImmutableList; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.SqlTypeName; import org.junit.Assert; @@ -173,4 +174,60 @@ private SourceConfig mockSourceConfig(String alias) { return new SourceConfig( GraphOpt.Source.VERTEX, new LabelConfig(false).addLabel("person"), alias); } + + @Test + public void label_1_test() { + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX, new LabelConfig(true), "a")) + .expand(new ExpandConfig(GraphOpt.Expand.OUT, new LabelConfig(true), "b")) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.literal("age"), + builder.variable("b", "weight"))) + .getV(new GetVConfig(GraphOpt.GetV.END)) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.literal("software"), + builder.variable(null, GraphProperty.LABEL_KEY))) + .build(); + System.out.println(node.explain()); + // System.out.println(node.getRowType()); + } + +// @Test +// public void label_2_test() { +// RexNode var = +// builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) +// .variable(null, GraphProperty.LABEL_KEY); +// RexNode inNode = +// builder.getRexBuilder().makeIn(var, ImmutableList.of(builder.literal("person"))); +// RelNode node = +// builder.filter(inNode) +// .filter( +// builder.getRexBuilder() +// .makeIn(var, ImmutableList.of(builder.literal("software")))) +// .build(); +// System.out.println(node.explain()); +// } + + @Test + public void label_3_test() { + RexNode var = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + .variable(null, GraphProperty.ID_KEY); + RexNode inNode = + builder.getRexBuilder() + .makeIn(var, ImmutableList.of(builder.literal(1), builder.literal(2))); + RelNode node = + builder.filter(inNode) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, "name"), + builder.literal("marko"))) + .build(); + System.out.println(node.explain()); + } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java new file mode 100644 index 000000000000..53fbb18e08f6 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4x; + +import com.alibaba.graphscope.common.ir.Utils; +import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.gremlin.antlr4x.parser.GremlinAntlr4Parser; +import com.alibaba.graphscope.gremlin.antlr4x.visitor.GraphBuilderVisitor; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.calcite.rel.RelNode; +import org.junit.Test; + +public class GraphBuilderTest { + @Test + public void g_V_test() { + GraphBuilder builder = Utils.mockGraphBuilder(); + GraphBuilderVisitor visitor = new GraphBuilderVisitor(builder); + ParseTree parseTree = + new GremlinAntlr4Parser() + .parse("g.V().hasLabel('person').hasId(1, 2, 3).hasNot('name')"); + RelNode node = visitor.visit(parseTree).build(); + System.out.println(node.explain()); + } +} From d214db819a44b7141ea31053c42bf4310a832a8b Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 26 Sep 2023 10:59:32 +0800 Subject: [PATCH 03/10] [GIE Compiler] support expand and getV --- .../rel/graph/AbstractBindableTableScan.java | 2 +- .../common/ir/runtime/proto/Utils.java | 2 +- .../common/ir/tools/GraphBuilder.java | 43 +++-- .../common/ir/type/GraphSchemaType.java | 4 +- .../common/ir/type/GraphTypeFactoryImpl.java | 10 +- .../antlr4x/visitor/GraphBuilderVisitor.java | 182 +++++++++++++++++- .../visitor/PathExpandBuilderVisitor.java | 168 ++++++++++++++++ .../script/AntlrGremlinScriptEngine.java | 11 +- .../graphscope/common/ir/ExpressionTest.java | 116 ++++++++--- .../gremlin/antlr4x/GraphBuilderTest.java | 11 ++ 10 files changed, 488 insertions(+), 61 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java index 30b243693806..ba61a19b188e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/AbstractBindableTableScan.java @@ -95,7 +95,7 @@ public RelDataType deriveRowType() { for (RelOptTable table : tables) { GraphSchemaType type = (GraphSchemaType) table.getRowType(); // flat fuzzy labels to the list - tableTypes.addAll(type.getSchemaTypes()); + tableTypes.addAll(type.getSchemaTypeAsList()); } ObjectUtils.requireNonEmpty(tableTypes); GraphSchemaType graphType = diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index 8d049b15dec3..db7711381cbb 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -244,7 +244,7 @@ public static final DataType.IrDataType protoIrDataType( builder.setElementOpt( protoElementOpt(((GraphSchemaType) dataType).getScanOpt())); ((GraphSchemaType) dataType) - .getSchemaTypes() + .getSchemaTypeAsList() .forEach( k -> builder.addGraphDataType(protoElementType(k, isColumnId))); return DataType.IrDataType.newBuilder().setGraphType(builder.build()).build(); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index 9bbf8a4d6e41..f32966f2c321 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -700,7 +700,6 @@ public GraphBuilder filter(Iterable conditions) { private AbstractBindableTableScan fuseFilters( AbstractBindableTableScan tableScan, RexNode condition) { - ImmutableList originalFilters = tableScan.getFilters(); List labelValues = Lists.newArrayList(); List uniqueKeyFilters = Lists.newArrayList(); List extraFilters = Lists.newArrayList(); @@ -766,7 +765,14 @@ public Void visitInputRef(RexInputRef inputRef) { return null; } }; - if (!ObjectUtils.isEmpty(originalFilters)) { + if (tableScan instanceof GraphLogicalSource) { + GraphLogicalSource source = (GraphLogicalSource) tableScan; + if (source.getUniqueKeyFilters() != null) { + builder.filter(source.getUniqueKeyFilters()); + } + } + ImmutableList originalFilters = tableScan.getFilters(); + if (ObjectUtils.isNotEmpty(originalFilters)) { originalFilters.forEach(k -> ((RexNode) k).accept(propertyChecker)); builder.filter(originalFilters); } @@ -782,21 +788,20 @@ public Void visitInputRef(RexInputRef inputRef) { if (source.getUniqueKeyFilters() != null) { uniqueKeyFilters.add(0, source.getUniqueKeyFilters()); } + // todo: check if unique key filters have common keys, otherwise throw errors source.setUniqueKeyFilters( RexUtil.composeConjunction(this.getRexBuilder(), uniqueKeyFilters)); } if (!extraFilters.isEmpty()) { - if (ObjectUtils.isEmpty(originalFilters)) { - tableScan.setFilters( - ImmutableList.of( - RexUtil.composeConjunction(this.getRexBuilder(), extraFilters))); - } else { - List combineFilters = Lists.newArrayList(originalFilters); - combineFilters.addAll(extraFilters); - tableScan.setFilters( - ImmutableList.of( - RexUtil.composeConjunction(this.getRexBuilder(), combineFilters))); + ImmutableList originalFilters = tableScan.getFilters(); + if (ObjectUtils.isNotEmpty(originalFilters)) { + for (int i = 0; i < originalFilters.size(); ++i) { + extraFilters.add(i, (RexNode) originalFilters.get(i)); + } } + tableScan.setFilters( + ImmutableList.of( + RexUtil.composeConjunction(this.getRexBuilder(), extraFilters))); } return tableScan; } @@ -817,26 +822,24 @@ private void classifyFilters( RexNode left = rexCall.getOperands().get(0); RexNode right = rexCall.getOperands().get(1); if (left.getType() instanceof GraphLabelType && right instanceof RexLiteral) { - filtersToRemove.add(condition); + filtersToRemove.add(conjunction); labelValues.addAll( getValuesAsList(((RexLiteral) right).getValueAs(Comparable.class))); break; } else if (left instanceof RexLiteral && right.getType() instanceof GraphLabelType) { - filtersToRemove.add(condition); + filtersToRemove.add(conjunction); labelValues.addAll( getValuesAsList(((RexLiteral) left).getValueAs(Comparable.class))); break; } if (tableScan instanceof GraphLogicalSource) { if (isUniqueKey(left) && right instanceof RexLiteral) { - filtersToRemove.add(condition); - uniqueKeyFilters.add(condition); - break; + filtersToRemove.add(conjunction); + uniqueKeyFilters.add(conjunction); } else if (left instanceof RexLiteral && isUniqueKey(right)) { - filtersToRemove.add(condition); - uniqueKeyFilters.add(condition); - break; + filtersToRemove.add(conjunction); + uniqueKeyFilters.add(conjunction); } } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java index 60268d6960f8..acd4e33df0c2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java @@ -185,13 +185,13 @@ public RelDataTypeFamily getFamily() { return scanOpt; } - public List getSchemaTypes() { + public List getSchemaTypeAsList() { return ObjectUtils.isEmpty(this.fuzzySchemaTypes) ? ImmutableList.of(this) : Collections.unmodifiableList(this.fuzzySchemaTypes); } public boolean fuzzy() { - return this.labelType.getLabelsEntry().size() > 1; + return this.labelType.getLabelsEntry().size() > 1 || this.fuzzySchemaTypes.size() > 1; } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java index 4d09fbb8f4c8..b1b96fb0b9c4 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java @@ -37,8 +37,14 @@ public RelDataType createTypeWithNullability(RelDataType type, boolean nullable) RelDataType newType; if (type instanceof GraphSchemaType) { GraphSchemaType schemaType = (GraphSchemaType) type; - if (schemaType.getSchemaTypes().size() > 1) { - newType = GraphSchemaType.create(schemaType.getSchemaTypes(), this, nullable); + if (schemaType.getSchemaTypeAsList().size() > 1) { // fuzzy schema type + newType = + new GraphSchemaType( + schemaType.getScanOpt(), + schemaType.getLabelType(), + schemaType.getFieldList(), + schemaType.getSchemaTypeAsList(), + nullable); } else { newType = new GraphSchemaType( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java index c8d0d18893f3..00ddc90e0301 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java @@ -16,23 +16,30 @@ package com.alibaba.graphscope.gremlin.antlr4x.visitor; +import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalExpand; +import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalPathExpand; +import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; -import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; -import com.alibaba.graphscope.common.ir.tools.config.SourceConfig; +import com.alibaba.graphscope.common.ir.tools.config.*; import com.alibaba.graphscope.common.ir.type.GraphProperty; +import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.grammar.GremlinGSBaseVisitor; import com.alibaba.graphscope.grammar.GremlinGSParser; import com.alibaba.graphscope.gremlin.antlr4.GenericLiteralVisitor; +import com.alibaba.graphscope.gremlin.exception.InvalidGremlinScriptException; import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; - +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.ObjectUtils; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; public class GraphBuilderVisitor extends GremlinGSBaseVisitor { private final GraphBuilder builder; @@ -180,4 +187,173 @@ public GraphBuilder visitTraversalMethod_hasNot( return builder.filter( builder.call(GraphStdOperatorTable.IS_NULL, builder.variable(null, propertyKey))); } + + @Override + public GraphBuilder visitTraversalMethod_outE(GremlinGSParser.TraversalMethod_outEContext ctx) { + return builder.expand( + new ExpandConfig(GraphOpt.Expand.OUT, getLabelConfig(ctx.stringLiteralList()))); + } + + @Override + public GraphBuilder visitTraversalMethod_inE(GremlinGSParser.TraversalMethod_inEContext ctx) { + return builder.expand( + new ExpandConfig(GraphOpt.Expand.IN, getLabelConfig(ctx.stringLiteralList()))); + } + + @Override + public GraphBuilder visitTraversalMethod_bothE( + GremlinGSParser.TraversalMethod_bothEContext ctx) { + return builder.expand( + new ExpandConfig(GraphOpt.Expand.BOTH, getLabelConfig(ctx.stringLiteralList()))); + } + + @Override + public GraphBuilder visitTraversalMethod_outV(GremlinGSParser.TraversalMethod_outVContext ctx) { + return builder.getV(new GetVConfig(GraphOpt.GetV.START)); + } + + @Override + public GraphBuilder visitTraversalMethod_inV(GremlinGSParser.TraversalMethod_inVContext ctx) { + return builder.getV(new GetVConfig(GraphOpt.GetV.END)); + } + + @Override + public GraphBuilder visitTraversalMethod_otherV( + GremlinGSParser.TraversalMethod_otherVContext ctx) { + return builder.getV(new GetVConfig(GraphOpt.GetV.OTHER)); + } + + @Override + public GraphBuilder visitTraversalMethod_endV(GremlinGSParser.TraversalMethod_endVContext ctx) { + RelNode peek = builder.peek(); + if (peek instanceof GraphLogicalPathExpand) { + GraphLogicalPathExpand pathExpand = (GraphLogicalPathExpand) peek; + GraphLogicalExpand expand = (GraphLogicalExpand) pathExpand.getExpand(); + switch (expand.getOpt()) { + case OUT: + return builder.getV(new GetVConfig(GraphOpt.GetV.END)); + case IN: + return builder.getV(new GetVConfig(GraphOpt.GetV.START)); + case BOTH: + default: + return builder.getV(new GetVConfig(GraphOpt.GetV.OTHER)); + } + } + throw new InvalidGremlinScriptException("endV should follow with path expand"); + } + + @Override + public GraphBuilder visitTraversalMethod_out(GremlinGSParser.TraversalMethod_outContext ctx) { + if(pathExpandPattern(ctx.stringLiteralList())){ + return builder.pathExpand(new PathExpandBuilderVisitor(this).visitTraversalMethod_out(ctx).build()); + } else { + return builder.expand(new ExpandConfig(GraphOpt.Expand.OUT, getLabelConfig(ctx.stringLiteralList()))) + .getV(new GetVConfig(GraphOpt.GetV.END)); + } + } + + @Override + public GraphBuilder visitTraversalMethod_in(GremlinGSParser.TraversalMethod_inContext ctx) { + if(pathExpandPattern(ctx.stringLiteralList())){ + return builder.pathExpand(new PathExpandBuilderVisitor(this).visitTraversalMethod_in(ctx).build()); + } else { + return builder.expand(new ExpandConfig(GraphOpt.Expand.IN, getLabelConfig(ctx.stringLiteralList()))) + .getV(new GetVConfig(GraphOpt.GetV.START)); + } + } + + @Override + public GraphBuilder visitTraversalMethod_both(GremlinGSParser.TraversalMethod_bothContext ctx) { + if(pathExpandPattern(ctx.stringLiteralList())){ + return builder.pathExpand(new PathExpandBuilderVisitor(this).visitTraversalMethod_both(ctx).build()); + } else { + return builder.expand(new ExpandConfig(GraphOpt.Expand.BOTH, getLabelConfig(ctx.stringLiteralList()))) + .getV(new GetVConfig(GraphOpt.GetV.OTHER)); + } + } + + @Override + public GraphBuilder visitTraversalMethod_with(GremlinGSParser.TraversalMethod_withContext ctx) { + return builder; + } + + @Override + public GraphBuilder visitTraversalMethod_valueMap(GremlinGSParser.TraversalMethod_valueMapContext ctx) { + String[] properties = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + List propertyList = Lists.newArrayList(); + if (properties == null || properties.length == 0) { + RexGraphVariable curVar = builder.variable((String) null); + RelDataType dataType = curVar.getType(); + Preconditions.checkArgument(dataType instanceof GraphSchemaType, "can not get property from type=", dataType); + dataType.getFieldList().forEach(k -> propertyList.add(k.getName())); + } else { + for (int i = 0; i < properties.length; ++i) { + propertyList.add(properties[i]); + } + } + RexNode expr = builder.call(GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + propertyList.stream().flatMap(k -> + Stream.of( + builder.literal(k), + builder.variable(null, k))).collect(Collectors.toList())); + return builder.project(expr); + } + + @Override + public GraphBuilder visitTraversalMethod_values(GremlinGSParser.TraversalMethod_valuesContext ctx) { + if (ctx.getChildCount() == 4 && ctx.stringLiteral() != null) { + RexNode expr = builder.variable(null, GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); + return builder.project(expr); + } + throw new UnsupportedEvalException(ctx.getClass(), "supported pattern is [values('..')]"); + } + + @Override + public GraphBuilder visitTraversalMethod_elementMap( + GremlinGSParser.TraversalMethod_elementMapContext ctx) { + String[] properties = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + List propertyList = Lists.newArrayList(GraphProperty.LABEL_KEY, GraphProperty.ID_KEY); + if (properties == null || properties.length == 0) { + RexGraphVariable curVar = builder.variable((String) null); + RelDataType dataType = curVar.getType(); + Preconditions.checkArgument(dataType instanceof GraphSchemaType, "can not get property from type=", dataType); + dataType.getFieldList().forEach(k -> propertyList.add(k.getName())); + } else { + for (String property : properties) { + propertyList.add(property); + } + } + RexNode expr = builder.call(GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + propertyList.stream().flatMap(k -> + Stream.of( + builder.literal(k), + builder.variable(null, k))).collect(Collectors.toList())); + return builder.project(expr); + } + + public GraphBuilder getGraphBuilder() { + return this.builder; + } + + private boolean pathExpandPattern(GremlinGSParser.StringLiteralListContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx); + return labels != null && labels.length > 0 && rangeExpression(labels[0]); + } + + private boolean rangeExpression(String label) { + return label.matches("^\\d+\\.\\.\\d+"); + } + + private LabelConfig getLabelConfig(GremlinGSParser.StringLiteralListContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx); + if (labels == null || labels.length == 0) { + return new LabelConfig(true); + } else { + LabelConfig labelConfig = new LabelConfig(false); + for (int i = 0; i < labels.length; ++i) { + labelConfig.addLabel(labels[i]); + } + return labelConfig; + } + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java new file mode 100644 index 000000000000..a23b30dee966 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java @@ -0,0 +1,168 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4x.visitor; + +import com.alibaba.graphscope.common.ir.tools.config.*; +import com.alibaba.graphscope.grammar.GremlinGSBaseVisitor; +import com.alibaba.graphscope.grammar.GremlinGSParser; +import com.alibaba.graphscope.gremlin.Utils; +import com.alibaba.graphscope.gremlin.antlr4.GenericLiteralVisitor; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import org.antlr.v4.runtime.tree.ParseTree; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +public class PathExpandBuilderVisitor extends GremlinGSBaseVisitor { + private final GraphBuilderVisitor parent; + private final PathExpandConfig.Builder builder; + + public PathExpandBuilderVisitor(GraphBuilderVisitor parent) { + this.parent = Objects.requireNonNull(parent); + // PATH_OPT = ARBITRARY and RESULT_OPT = END_V are set by default + this.builder = PathExpandConfig.newBuilder(parent.getGraphBuilder()); + } + + @Override + public PathExpandConfig.Builder visitTraversalMethod_out( + GremlinGSParser.TraversalMethod_outContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + Preconditions.checkArgument( + labels != null && labels.length > 0, "arguments can not be empty in path expand"); + String[] ranges = labels[0].split("\\.\\."); + int lower = Integer.valueOf(ranges[0]); + int upper = Integer.valueOf(ranges[1]); + // set path_opt and result_opt + List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); + nextWith.forEach(k -> visitTraversalMethod_with(k)); + // set expand config, getV config, range + return builder.expand(new ExpandConfig(GraphOpt.Expand.OUT, getExpandLabelConfig(labels))) + .getV(new GetVConfig(GraphOpt.GetV.END)) + .range(lower, upper - lower); + } + + @Override + public PathExpandConfig.Builder visitTraversalMethod_in( + GremlinGSParser.TraversalMethod_inContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + Preconditions.checkArgument( + labels != null && labels.length > 0, "arguments can not be empty in path expand"); + String[] ranges = labels[0].split("\\.\\."); + int lower = Integer.valueOf(ranges[0]); + int upper = Integer.valueOf(ranges[1]); + // set path_opt and result_opt + List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); + nextWith.forEach(k -> visitTraversalMethod_with(k)); + // set expand config, getV config, range + return builder.expand(new ExpandConfig(GraphOpt.Expand.IN, getExpandLabelConfig(labels))) + .getV(new GetVConfig(GraphOpt.GetV.START)) + .range(lower, upper - lower); + } + + @Override + public PathExpandConfig.Builder visitTraversalMethod_both( + GremlinGSParser.TraversalMethod_bothContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + Preconditions.checkArgument( + labels != null && labels.length > 0, "arguments can not be empty in path expand"); + String[] ranges = labels[0].split("\\.\\."); + int lower = Integer.valueOf(ranges[0]); + int upper = Integer.valueOf(ranges[1]); + // set path_opt and result_opt + List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); + nextWith.forEach(k -> visitTraversalMethod_with(k)); + // set expand config, getV config, range + return builder.expand(new ExpandConfig(GraphOpt.Expand.BOTH, getExpandLabelConfig(labels))) + .getV(new GetVConfig(GraphOpt.GetV.OTHER)) + .range(lower, upper - lower); + } + + @Override + public PathExpandConfig.Builder visitTraversalMethod_with( + GremlinGSParser.TraversalMethod_withContext ctx) { + String optKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()); + Object optValue = GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral()); + switch (optKey.toUpperCase()) { + case "PATH_OPT": + return builder.pathOpt(GraphOpt.PathExpandPath.valueOf(String.valueOf(optValue).toUpperCase())); + case "RESULT_OPT": + return builder.resultOpt(GraphOpt.PathExpandResult.valueOf(String.valueOf(optValue).toUpperCase())); + default: + return builder; + } + } + + private List getNextWith(GremlinGSParser.TraversalMethodContext curCtx) { + List nextWith = Lists.newArrayList(); + TraversalMethodIterator methodIterator = new TraversalMethodIterator(curCtx); + while(methodIterator.hasNext()) { + GremlinGSParser.TraversalMethodContext next = methodIterator.next(); + if (next.traversalMethod_with() != null) { + nextWith.add(next.traversalMethod_with()); + } else { + break; + } + } + return nextWith; + } + + + private LabelConfig getExpandLabelConfig(String[] labels) { + if (labels.length <= 1) { + return new LabelConfig(true); + } else { + labels = Utils.removeStringEle(0, labels); + LabelConfig expandLabels = new LabelConfig(false); + for (int i = 0; i < labels.length; ++i) { + expandLabels.addLabel(labels[i]); + } + return expandLabels; + } + } + + private static class TraversalMethodIterator implements Iterator { + private GremlinGSParser.TraversalMethodContext _next; + public TraversalMethodIterator(GremlinGSParser.TraversalMethodContext start) { + this._next = start; + } + + @Override + public boolean hasNext() { + ParseTree parent = getParent(getParent(_next)); + return parent != null && parent.getChildCount() >= 3; + } + + @Override + public GremlinGSParser.TraversalMethodContext next() { + ParseTree parent = getParent(getParent(_next)); + if (parent == null || parent.getChildCount() < 3) return null; + _next = (GremlinGSParser.TraversalMethodContext) parent.getChild(2); + return _next; + } + + private @Nullable ParseTree getParent(ParseTree child) { + Class parentClass = GremlinGSParser.ChainedTraversalContext.class; + while (child != null && child.getParent() != null && !child.getParent().getClass().equals(parentClass)) { + child = child.getParent(); + } + return (child != null && child.getParent() != null && child.getParent().getClass().equals(parentClass)) ? child.getParent() : null; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java index dbf8ff49f84d..2b2ecf543f3c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java @@ -29,8 +29,9 @@ import com.alibaba.graphscope.grammar.GremlinGSLexer; import com.alibaba.graphscope.grammar.GremlinGSParser; import com.alibaba.graphscope.gremlin.antlr4.GremlinAntlrToJava; - -import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; import org.antlr.v4.runtime.atn.PredictionMode; import org.apache.commons.lang3.NotImplementedException; import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; @@ -41,10 +42,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.script.AbstractScriptEngine; +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.SimpleBindings; import java.io.Reader; -import javax.script.*; - public class AntlrGremlinScriptEngine extends AbstractScriptEngine implements GremlinScriptEngine { private Logger logger = LoggerFactory.getLogger(AntlrGremlinScriptEngine.class); private volatile AntlrGremlinScriptEngineFactory factory; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java index 96ca190cf6d8..d6887455db27 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java @@ -170,50 +170,110 @@ public void unary_minus_test() { Assert.assertEquals("-(a.age)", node.toString()); } - private SourceConfig mockSourceConfig(String alias) { - return new SourceConfig( - GraphOpt.Source.VERTEX, new LabelConfig(false).addLabel("person"), alias); + @Test + public void label_1_test() { + RelNode node = + builder.source(new SourceConfig(GraphOpt.Source.VERTEX, new LabelConfig(true), "a")) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.literal("software"), + builder.variable(null, GraphProperty.LABEL_KEY))) + .build(); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=false, tables=[software]}], alias=[a]," + + " opt=[VERTEX])", + node.explain().trim()); } + // check property after updating the label type @Test - public void label_1_test() { + public void label_2_test() { + try { + RelNode node = + builder.source( + new SourceConfig( + GraphOpt.Source.VERTEX, new LabelConfig(true), "a")) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, "age"), + builder.literal(1))) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.literal("software"), + builder.variable(null, GraphProperty.LABEL_KEY))) + .build(); + } catch (IllegalArgumentException e) { + Assert.assertEquals( + "{property=age} not found; expected properties are: [id, name, lang," + + " creationDate]", + e.getMessage()); + return; + } + Assert.fail(); + } + + @Test + public void label_3_test() { RelNode node = builder.source(new SourceConfig(GraphOpt.Source.VERTEX, new LabelConfig(true), "a")) - .expand(new ExpandConfig(GraphOpt.Expand.OUT, new LabelConfig(true), "b")) .filter( builder.call( GraphStdOperatorTable.EQUALS, - builder.literal("age"), - builder.variable("b", "weight"))) - .getV(new GetVConfig(GraphOpt.GetV.END)) + builder.variable(null, GraphProperty.ID_KEY), + builder.literal(1))) .filter( builder.call( GraphStdOperatorTable.EQUALS, builder.literal("software"), builder.variable(null, GraphProperty.LABEL_KEY))) + .filter( + builder.call( + GraphStdOperatorTable.AND, + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, "~id"), + builder.literal(3)), + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.ID_KEY), + builder.literal(2)))) + .filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, "lang"), + builder.literal("a"))) .build(); - System.out.println(node.explain()); - // System.out.println(node.getRowType()); - } - -// @Test -// public void label_2_test() { -// RexNode var = -// builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) -// .variable(null, GraphProperty.LABEL_KEY); -// RexNode inNode = -// builder.getRexBuilder().makeIn(var, ImmutableList.of(builder.literal("person"))); -// RelNode node = -// builder.filter(inNode) -// .filter( -// builder.getRexBuilder() -// .makeIn(var, ImmutableList.of(builder.literal("software")))) -// .build(); -// System.out.println(node.explain()); -// } + System.out.println(node.explain().trim()); + } + + private SourceConfig mockSourceConfig(String alias) { + return new SourceConfig( + GraphOpt.Source.VERTEX, new LabelConfig(false).addLabel("person"), alias); + } + + // @Test + // public void label_2_test() { + // RexNode var = + // builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) + // .variable(null, GraphProperty.LABEL_KEY); + // RexNode inNode = + // builder.getRexBuilder().makeIn(var, + // ImmutableList.of(builder.literal("person"))); + // RelNode node = + // builder.filter(inNode) + // .filter( + // builder.getRexBuilder() + // .makeIn(var, + // ImmutableList.of(builder.literal("software")))) + // .build(); + // System.out.println(node.explain()); + // } @Test - public void label_3_test() { + public void label_4_test() { RexNode var = builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) .variable(null, GraphProperty.ID_KEY); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java index 53fbb18e08f6..e7c42c2a2e7e 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java @@ -36,4 +36,15 @@ public void g_V_test() { RelNode node = visitor.visit(parseTree).build(); System.out.println(node.explain()); } + + @Test + public void g_V_out_test() { + GraphBuilder builder = Utils.mockGraphBuilder(); + GraphBuilderVisitor visitor = new GraphBuilderVisitor(builder); + ParseTree parseTree = + new GremlinAntlr4Parser() + .parse("g.V().hasLabel('person').in('1..2', 'knows').with('result_opt', 'all_v').endV()"); + RelNode node = visitor.visit(parseTree).build(); + System.out.println(node.explain()); + } } From b8d2a87b8e7fc319c2db96a57397bb2ab0fef216 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 26 Sep 2023 19:44:56 +0800 Subject: [PATCH 04/10] [GIE Compiler] fix map value constructor --- .../operator/SqlArrayValueConstructor.java | 24 +++++++++------ .../rex/operator/SqlMapValueConstructor.java | 30 +++++++++---------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java index ef56370b4c65..f571b62fe7dc 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java @@ -24,6 +24,7 @@ import org.apache.calcite.sql.fun.SqlMultisetValueConstructor; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; +import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; @@ -36,15 +37,7 @@ public SqlArrayValueConstructor() { public RelDataType inferReturnType(SqlOperatorBinding opBinding) { RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); List argTypes = opBinding.collectOperandTypes(); - RelDataType componentType; - if (argTypes.isEmpty()) { - componentType = typeFactory.createSqlType(SqlTypeName.ANY); - } else { - componentType = getComponentType(typeFactory, argTypes); - if (componentType == null) { - componentType = typeFactory.createSqlType(SqlTypeName.ANY); - } - } + RelDataType componentType = getComponentType(typeFactory, argTypes); return SqlTypeUtil.createArrayType(typeFactory, componentType, false); } @@ -53,4 +46,17 @@ public RelDataType inferReturnType(SqlOperatorBinding opBinding) { public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { return true; } + + @Override + protected @Nullable RelDataType getComponentType( + RelDataTypeFactory typeFactory, List argTypes) { + try { + RelDataType componentType = typeFactory.leastRestrictive(argTypes); + return (componentType == null) + ? typeFactory.createSqlType(SqlTypeName.ANY) + : componentType; + } catch (Exception e) { + return typeFactory.createSqlType(SqlTypeName.ANY); + } + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java index 4bedfa411d26..87fc5e5a2a48 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java @@ -37,8 +37,12 @@ public SqlMapValueConstructor() { } public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + List argTypes = opBinding.collectOperandTypes(); Pair type = - getComponentTypes(opBinding.getTypeFactory(), opBinding.collectOperandTypes()); + Pair.of( + getComponentType(typeFactory, Util.quotientList(argTypes, 2, 0)), + getComponentType(typeFactory, Util.quotientList(argTypes, 2, 1))); return SqlTypeUtil.createMapType(opBinding.getTypeFactory(), type.left, type.right, false); } @@ -51,22 +55,16 @@ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFail return true; } - private static Pair<@Nullable RelDataType, @Nullable RelDataType> getComponentTypes( + @Override + protected @Nullable RelDataType getComponentType( RelDataTypeFactory typeFactory, List argTypes) { - RelDataType leftType, rightType; - if (argTypes.isEmpty()) { - leftType = typeFactory.createSqlType(SqlTypeName.ANY); - rightType = typeFactory.createSqlType(SqlTypeName.ANY); - } else { - leftType = typeFactory.leastRestrictive(Util.quotientList(argTypes, 2, 0)); - rightType = typeFactory.leastRestrictive(Util.quotientList(argTypes, 2, 1)); - if (leftType == null) { - leftType = typeFactory.createSqlType(SqlTypeName.ANY); - } - if (rightType == null) { - rightType = typeFactory.createSqlType(SqlTypeName.ANY); - } + try { + RelDataType componentType = typeFactory.leastRestrictive(argTypes); + return (componentType == null) + ? typeFactory.createSqlType(SqlTypeName.ANY) + : componentType; + } catch (Exception e) { + return typeFactory.createSqlType(SqlTypeName.ANY); } - return Pair.of(leftType, rightType); } } From 5072ebc51880872a28513c160222ba50f17ddcc3 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 26 Sep 2023 19:45:52 +0800 Subject: [PATCH 05/10] minor fix --- .../operator/SqlArrayValueConstructor.java | 56 +++ .../rex/operator/SqlMapValueConstructor.java | 72 ++++ .../common/ir/tools/GraphBuilder.java | 1 + .../ir/tools/GraphStdOperatorTable.java | 14 +- .../common/ir/type/GraphSchemaType.java | 1 - .../antlr4x/visitor/GraphBuilderVisitor.java | 374 +++++++++++++++--- .../visitor/PathExpandBuilderVisitor.java | 56 +-- .../visitor/TraversalMethodIterator.java | 87 ++++ .../gremlin/antlr4x/GraphBuilderTest.java | 7 +- 9 files changed, 554 insertions(+), 114 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/TraversalMethodIterator.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java new file mode 100644 index 000000000000..ef56370b4c65 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.rex.operator; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.fun.SqlMultisetValueConstructor; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; + +import java.util.List; + +public class SqlArrayValueConstructor extends SqlMultisetValueConstructor { + public SqlArrayValueConstructor() { + super("ARRAY", SqlKind.ARRAY_VALUE_CONSTRUCTOR); + } + + @Override + public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + List argTypes = opBinding.collectOperandTypes(); + RelDataType componentType; + if (argTypes.isEmpty()) { + componentType = typeFactory.createSqlType(SqlTypeName.ANY); + } else { + componentType = getComponentType(typeFactory, argTypes); + if (componentType == null) { + componentType = typeFactory.createSqlType(SqlTypeName.ANY); + } + } + return SqlTypeUtil.createArrayType(typeFactory, componentType, false); + } + + // operands of array value constructor can be any, even if empty, i.e [] + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + return true; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java new file mode 100644 index 000000000000..4bedfa411d26 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.rex.operator; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.fun.SqlMultisetValueConstructor; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.util.Pair; +import org.apache.calcite.util.Static; +import org.apache.calcite.util.Util; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +public class SqlMapValueConstructor extends SqlMultisetValueConstructor { + public SqlMapValueConstructor() { + super("MAP", SqlKind.MAP_VALUE_CONSTRUCTOR); + } + + public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + Pair type = + getComponentTypes(opBinding.getTypeFactory(), opBinding.collectOperandTypes()); + return SqlTypeUtil.createMapType(opBinding.getTypeFactory(), type.left, type.right, false); + } + + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + List argTypes = callBinding.collectOperandTypes(); + if (argTypes.size() % 2 > 0) { + throw callBinding.newValidationError(Static.RESOURCE.mapRequiresEvenArgCount()); + } + return true; + } + + private static Pair<@Nullable RelDataType, @Nullable RelDataType> getComponentTypes( + RelDataTypeFactory typeFactory, List argTypes) { + RelDataType leftType, rightType; + if (argTypes.isEmpty()) { + leftType = typeFactory.createSqlType(SqlTypeName.ANY); + rightType = typeFactory.createSqlType(SqlTypeName.ANY); + } else { + leftType = typeFactory.leastRestrictive(Util.quotientList(argTypes, 2, 0)); + rightType = typeFactory.leastRestrictive(Util.quotientList(argTypes, 2, 1)); + if (leftType == null) { + leftType = typeFactory.createSqlType(SqlTypeName.ANY); + } + if (rightType == null) { + rightType = typeFactory.createSqlType(SqlTypeName.ANY); + } + } + return Pair.of(leftType, rightType); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index f32966f2c321..061c42ec6033 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -646,6 +646,7 @@ private boolean isCurrentSupported(SqlOperator operator) { || (sqlKind == SqlKind.PROCEDURE_CALL) || (sqlKind == SqlKind.NOT) || sqlKind == SqlKind.ARRAY_VALUE_CONSTRUCTOR + || sqlKind == SqlKind.MAP_VALUE_CONSTRUCTOR || sqlKind == SqlKind.IS_NULL || sqlKind == SqlKind.IS_NOT_NULL || sqlKind == SqlKind.EXTRACT diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java index 1959fa4571fb..1b5402d39b6a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java @@ -18,6 +18,8 @@ import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; import com.alibaba.graphscope.common.ir.rex.operator.CaseOperator; +import com.alibaba.graphscope.common.ir.rex.operator.SqlArrayValueConstructor; +import com.alibaba.graphscope.common.ir.rex.operator.SqlMapValueConstructor; import org.apache.calcite.sql.*; import org.apache.calcite.sql.fun.SqlMonotonicBinaryOperator; @@ -208,15 +210,7 @@ public static final SqlFunction USER_DEFINED_PROCEDURE(StoredProcedureMeta meta) } // combine multiple expressions into a list - public static final SqlOperator ARRAY_VALUE_CONSTRUCTOR = - new SqlSpecialOperator( - "ARRAY_VALUE_CONSTRUCTOR", - SqlKind.ARRAY_VALUE_CONSTRUCTOR, - 0, - false, - ReturnTypes.explicit(SqlTypeName.ANY).andThen(SqlTypeTransforms.TO_ARRAY), - GraphInferTypes.FIRST_KNOWN, - OperandTypes.ANY); + public static final SqlOperator ARRAY_VALUE_CONSTRUCTOR = new SqlArrayValueConstructor(); public static final SqlFunction EXTRACT = new SqlFunction( @@ -226,4 +220,6 @@ public static final SqlFunction USER_DEFINED_PROCEDURE(StoredProcedureMeta meta) null, GraphOperandTypes.INTERVALINTERVAL_INTERVALDATETIME, SqlFunctionCategory.SYSTEM); + + public static final SqlOperator MAP_VALUE_CONSTRUCTOR = new SqlMapValueConstructor(); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java index acd4e33df0c2..c075a48377cc 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java @@ -20,7 +20,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; - import org.apache.calcite.linq4j.Ord; import org.apache.calcite.rel.type.*; import org.apache.commons.lang3.ObjectUtils; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java index 00ddc90e0301..6b5c2affb437 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java @@ -27,17 +27,25 @@ import com.alibaba.graphscope.grammar.GremlinGSBaseVisitor; import com.alibaba.graphscope.grammar.GremlinGSParser; import com.alibaba.graphscope.gremlin.antlr4.GenericLiteralVisitor; +import com.alibaba.graphscope.gremlin.antlr4.TraversalEnumParser; import com.alibaba.graphscope.gremlin.exception.InvalidGremlinScriptException; import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.ObjectUtils; +import org.apache.tinkerpop.gremlin.structure.Column; +import org.apache.tinkerpop.gremlin.structure.T; +import org.checkerframework.checker.nullness.qual.Nullable; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -51,13 +59,27 @@ public GraphBuilderVisitor(GraphBuilder builder) { @Override public GraphBuilder visitTraversalSourceSpawnMethod_V( GremlinGSParser.TraversalSourceSpawnMethod_VContext ctx) { - return builder.source(new SourceConfig(GraphOpt.Source.VERTEX)); + return builder.source( + new SourceConfig( + GraphOpt.Source.VERTEX, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalSourceSpawnMethodContext) + ctx.getParent())))); } @Override public GraphBuilder visitTraversalSourceSpawnMethod_E( GremlinGSParser.TraversalSourceSpawnMethod_EContext ctx) { - return builder.source(new SourceConfig(GraphOpt.Source.EDGE)); + return builder.source( + new SourceConfig( + GraphOpt.Source.EDGE, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalSourceSpawnMethodContext) + ctx.getParent())))); } @Override @@ -191,36 +213,75 @@ public GraphBuilder visitTraversalMethod_hasNot( @Override public GraphBuilder visitTraversalMethod_outE(GremlinGSParser.TraversalMethod_outEContext ctx) { return builder.expand( - new ExpandConfig(GraphOpt.Expand.OUT, getLabelConfig(ctx.stringLiteralList()))); + new ExpandConfig( + GraphOpt.Expand.OUT, + getLabelConfig(ctx.stringLiteralList()), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } @Override public GraphBuilder visitTraversalMethod_inE(GremlinGSParser.TraversalMethod_inEContext ctx) { return builder.expand( - new ExpandConfig(GraphOpt.Expand.IN, getLabelConfig(ctx.stringLiteralList()))); + new ExpandConfig( + GraphOpt.Expand.IN, + getLabelConfig(ctx.stringLiteralList()), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } @Override public GraphBuilder visitTraversalMethod_bothE( GremlinGSParser.TraversalMethod_bothEContext ctx) { return builder.expand( - new ExpandConfig(GraphOpt.Expand.BOTH, getLabelConfig(ctx.stringLiteralList()))); + new ExpandConfig( + GraphOpt.Expand.BOTH, + getLabelConfig(ctx.stringLiteralList()), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } @Override public GraphBuilder visitTraversalMethod_outV(GremlinGSParser.TraversalMethod_outVContext ctx) { - return builder.getV(new GetVConfig(GraphOpt.GetV.START)); + return builder.getV( + new GetVConfig( + GraphOpt.GetV.START, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } @Override public GraphBuilder visitTraversalMethod_inV(GremlinGSParser.TraversalMethod_inVContext ctx) { - return builder.getV(new GetVConfig(GraphOpt.GetV.END)); + return builder.getV( + new GetVConfig( + GraphOpt.GetV.END, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } @Override public GraphBuilder visitTraversalMethod_otherV( GremlinGSParser.TraversalMethod_otherVContext ctx) { - return builder.getV(new GetVConfig(GraphOpt.GetV.OTHER)); + return builder.getV( + new GetVConfig( + GraphOpt.GetV.OTHER, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } @Override @@ -229,14 +290,21 @@ public GraphBuilder visitTraversalMethod_endV(GremlinGSParser.TraversalMethod_en if (peek instanceof GraphLogicalPathExpand) { GraphLogicalPathExpand pathExpand = (GraphLogicalPathExpand) peek; GraphLogicalExpand expand = (GraphLogicalExpand) pathExpand.getExpand(); + String tag = + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); switch (expand.getOpt()) { case OUT: - return builder.getV(new GetVConfig(GraphOpt.GetV.END)); + return builder.getV( + new GetVConfig(GraphOpt.GetV.END, new LabelConfig(true), tag)); case IN: - return builder.getV(new GetVConfig(GraphOpt.GetV.START)); + return builder.getV( + new GetVConfig(GraphOpt.GetV.START, new LabelConfig(true), tag)); case BOTH: default: - return builder.getV(new GetVConfig(GraphOpt.GetV.OTHER)); + return builder.getV( + new GetVConfig(GraphOpt.GetV.OTHER, new LabelConfig(true), tag)); } } throw new InvalidGremlinScriptException("endV should follow with path expand"); @@ -244,31 +312,61 @@ public GraphBuilder visitTraversalMethod_endV(GremlinGSParser.TraversalMethod_en @Override public GraphBuilder visitTraversalMethod_out(GremlinGSParser.TraversalMethod_outContext ctx) { - if(pathExpandPattern(ctx.stringLiteralList())){ - return builder.pathExpand(new PathExpandBuilderVisitor(this).visitTraversalMethod_out(ctx).build()); + if (pathExpandPattern(ctx.stringLiteralList())) { + return builder.pathExpand( + new PathExpandBuilderVisitor(this).visitTraversalMethod_out(ctx).build()); } else { - return builder.expand(new ExpandConfig(GraphOpt.Expand.OUT, getLabelConfig(ctx.stringLiteralList()))) - .getV(new GetVConfig(GraphOpt.GetV.END)); + return builder.expand( + new ExpandConfig( + GraphOpt.Expand.OUT, getLabelConfig(ctx.stringLiteralList()))) + .getV( + new GetVConfig( + GraphOpt.GetV.END, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } } @Override public GraphBuilder visitTraversalMethod_in(GremlinGSParser.TraversalMethod_inContext ctx) { - if(pathExpandPattern(ctx.stringLiteralList())){ - return builder.pathExpand(new PathExpandBuilderVisitor(this).visitTraversalMethod_in(ctx).build()); + if (pathExpandPattern(ctx.stringLiteralList())) { + return builder.pathExpand( + new PathExpandBuilderVisitor(this).visitTraversalMethod_in(ctx).build()); } else { - return builder.expand(new ExpandConfig(GraphOpt.Expand.IN, getLabelConfig(ctx.stringLiteralList()))) - .getV(new GetVConfig(GraphOpt.GetV.START)); + return builder.expand( + new ExpandConfig( + GraphOpt.Expand.IN, getLabelConfig(ctx.stringLiteralList()))) + .getV( + new GetVConfig( + GraphOpt.GetV.START, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } } @Override public GraphBuilder visitTraversalMethod_both(GremlinGSParser.TraversalMethod_bothContext ctx) { - if(pathExpandPattern(ctx.stringLiteralList())){ - return builder.pathExpand(new PathExpandBuilderVisitor(this).visitTraversalMethod_both(ctx).build()); + if (pathExpandPattern(ctx.stringLiteralList())) { + return builder.pathExpand( + new PathExpandBuilderVisitor(this).visitTraversalMethod_both(ctx).build()); } else { - return builder.expand(new ExpandConfig(GraphOpt.Expand.BOTH, getLabelConfig(ctx.stringLiteralList()))) - .getV(new GetVConfig(GraphOpt.GetV.OTHER)); + return builder.expand( + new ExpandConfig( + GraphOpt.Expand.BOTH, getLabelConfig(ctx.stringLiteralList()))) + .getV( + new GetVConfig( + GraphOpt.GetV.OTHER, + new LabelConfig(true), + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) + ctx.getParent())))); } } @@ -278,32 +376,42 @@ public GraphBuilder visitTraversalMethod_with(GremlinGSParser.TraversalMethod_wi } @Override - public GraphBuilder visitTraversalMethod_valueMap(GremlinGSParser.TraversalMethod_valueMapContext ctx) { - String[] properties = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); - List propertyList = Lists.newArrayList(); - if (properties == null || properties.length == 0) { - RexGraphVariable curVar = builder.variable((String) null); - RelDataType dataType = curVar.getType(); - Preconditions.checkArgument(dataType instanceof GraphSchemaType, "can not get property from type=", dataType); - dataType.getFieldList().forEach(k -> propertyList.add(k.getName())); - } else { - for (int i = 0; i < properties.length; ++i) { - propertyList.add(properties[i]); - } - } - RexNode expr = builder.call(GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, - propertyList.stream().flatMap(k -> - Stream.of( - builder.literal(k), - builder.variable(null, k))).collect(Collectors.toList())); - return builder.project(expr); + public GraphBuilder visitTraversalMethod_as(GremlinGSParser.TraversalMethod_asContext ctx) { + return builder; } @Override - public GraphBuilder visitTraversalMethod_values(GremlinGSParser.TraversalMethod_valuesContext ctx) { + public GraphBuilder visitTraversalMethod_valueMap( + GremlinGSParser.TraversalMethod_valueMapContext ctx) { + RexNode expr = + builder.call( + GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + getProperties(ctx, null).stream() + .flatMap( + k -> + Stream.of( + builder.literal(k), + builder.variable(null, k))) + .collect(Collectors.toList())); + String tag = getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + return builder.project( + ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); + } + + @Override + public GraphBuilder visitTraversalMethod_values( + GremlinGSParser.TraversalMethod_valuesContext ctx) { if (ctx.getChildCount() == 4 && ctx.stringLiteral() != null) { - RexNode expr = builder.variable(null, GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); - return builder.project(expr); + RexNode expr = + builder.variable( + null, GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); + String tag = getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + return builder.project( + ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); } throw new UnsupportedEvalException(ctx.getClass(), "supported pattern is [values('..')]"); } @@ -311,30 +419,118 @@ public GraphBuilder visitTraversalMethod_values(GremlinGSParser.TraversalMethod_ @Override public GraphBuilder visitTraversalMethod_elementMap( GremlinGSParser.TraversalMethod_elementMapContext ctx) { - String[] properties = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); - List propertyList = Lists.newArrayList(GraphProperty.LABEL_KEY, GraphProperty.ID_KEY); - if (properties == null || properties.length == 0) { - RexGraphVariable curVar = builder.variable((String) null); - RelDataType dataType = curVar.getType(); - Preconditions.checkArgument(dataType instanceof GraphSchemaType, "can not get property from type=", dataType); - dataType.getFieldList().forEach(k -> propertyList.add(k.getName())); - } else { - for (String property : properties) { - propertyList.add(property); + RexNode expr = + builder.call( + GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + getProperties(ctx, null).stream() + .flatMap( + k -> + Stream.of( + builder.literal(k), + builder.variable(null, k))) + .collect(Collectors.toList())); + String tag = getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + return builder.project( + ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); + } + + @Override + public GraphBuilder visitTraversalMethod_select( + GremlinGSParser.TraversalMethod_selectContext ctx) { + String tag = getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + RexNode expr; + if (ctx.stringLiteral() != null) { + List selectTags = + Lists.newArrayList(GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); + if (ctx.stringLiteralList() != null) { + String[] tags = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + selectTags.addAll(Arrays.asList(tags)); } + GremlinGSParser.TraversalMethod_selectby_listContext listCtx = + ctx.traversalMethod_selectby_list(); + List byCtxs = + listCtx == null + ? ImmutableList.of() + : listCtx.getRuleContexts( + GremlinGSParser.TraversalMethod_selectbyContext.class); + Map keyValueMap = Maps.newLinkedHashMap(); + for (int i = 0; i < selectTags.size(); ++i) { + String selectTag = selectTags.get(i); + keyValueMap.put(selectTag, convertSelectByCtx(byCtxs, i, selectTag)); + } + Preconditions.checkArgument( + !keyValueMap.isEmpty(), "keyValue should not be empty in select"); + + if (keyValueMap.size() == 1) { + expr = keyValueMap.entrySet().iterator().next().getValue(); + } else { + List mapParameters = Lists.newArrayList(); + keyValueMap.forEach( + (k, v) -> { + mapParameters.add(builder.literal(k)); + mapParameters.add(v); + }); + expr = builder.call(GraphStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, mapParameters); + } + } else if (ctx.traversalColumn() != null) { + Column column = + TraversalEnumParser.parseTraversalEnumFromContext( + Column.class, ctx.traversalColumn()); + expr = builder.variable(column.name()); + } else { + throw new UnsupportedEvalException( + GremlinGSParser.TraversalMethod_selectbyContext.class, + ctx.getText() + " is unsupported yet"); } - RexNode expr = builder.call(GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, - propertyList.stream().flatMap(k -> - Stream.of( - builder.literal(k), - builder.variable(null, k))).collect(Collectors.toList())); - return builder.project(expr); + return builder.project(ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); } public GraphBuilder getGraphBuilder() { return this.builder; } + private RexNode convertSelectByCtx( + List byCtxs, int i, String tag) { + int ctxCnt = byCtxs.size(); + if (ctxCnt == 0) { + return builder.variable(tag); + } + GremlinGSParser.TraversalMethod_selectbyContext byCtx = byCtxs.get(i % ctxCnt); + int byChildCount = byCtx.getChildCount(); + if (byChildCount == 3) { // select(..).by() + return builder.variable(tag); + } else if (byChildCount == 4 && byCtx.stringLiteral() != null) { // select(..).by('name') + return builder.variable( + tag, GenericLiteralVisitor.getStringLiteral(byCtx.stringLiteral())); + } else if (byChildCount == 4 + && byCtx.traversalToken() != null) { // select(..).by(T.label/T.id) + T token = + TraversalEnumParser.parseTraversalEnumFromContext( + T.class, byCtx.traversalToken()); + return builder.variable(tag, token.getAccessor()); + } else if (byCtx.traversalMethod_valueMap() != null) { // select(..).by(valueMap('name')) + return builder.call( + GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + getProperties(byCtx.traversalMethod_valueMap(), tag).stream() + .flatMap(k -> Stream.of(builder.literal(k), builder.variable(tag, k))) + .collect(Collectors.toList())); + } else if (byCtx.traversalMethod_elementMap() + != null) { // select(..).by(elementMap('name')) + return builder.call( + GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + getProperties(byCtx.traversalMethod_elementMap(), tag).stream() + .flatMap(k -> Stream.of(builder.literal(k), builder.variable(tag, k))) + .collect(Collectors.toList())); + } + throw new UnsupportedEvalException( + GremlinGSParser.TraversalMethod_selectbyContext.class, + byCtx.getText() + " is unsupported yet in select"); + } + private boolean pathExpandPattern(GremlinGSParser.StringLiteralListContext ctx) { String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx); return labels != null && labels.length > 0 && rangeExpression(labels[0]); @@ -356,4 +552,60 @@ private LabelConfig getLabelConfig(GremlinGSParser.StringLiteralListContext ctx) return labelConfig; } } + + private List getProperties( + GremlinGSParser.TraversalMethod_valueMapContext ctx, @Nullable String tag) { + String[] properties = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + return (properties == null || properties.length == 0) + ? getAllProperties(tag) + : Arrays.asList(properties); + } + + private List getProperties( + GremlinGSParser.TraversalMethod_elementMapContext ctx, @Nullable String tag) { + String[] properties = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + List propertyList = + Lists.newArrayList(GraphProperty.LABEL_KEY, GraphProperty.ID_KEY); + if (properties == null || properties.length == 0) { + propertyList.addAll(getAllProperties(tag)); + } else { + propertyList.addAll(Arrays.asList(properties)); + } + return propertyList; + } + + private List getAllProperties(@Nullable String tag) { + RexGraphVariable curVar = builder.variable(tag); + RelDataType dataType = curVar.getType(); + Preconditions.checkArgument( + dataType instanceof GraphSchemaType, "can not get property from type=", dataType); + return dataType.getFieldList().stream().map(k -> k.getName()).collect(Collectors.toList()); + } + + protected @Nullable String getNextTag(TraversalMethodIterator methodIterator) { + List tags = Lists.newArrayList(); + while (methodIterator.hasNext()) { + GremlinGSParser.TraversalMethodContext next = methodIterator.next(); + if (next.traversalMethod_as() != null) { + tags.add( + GenericLiteralVisitor.getStringLiteral( + next.traversalMethod_as().stringLiteral())); + } else if (next.traversalMethod_has() != null + || next.traversalMethod_hasId() != null + || next.traversalMethod_hasLabel() != null + || next.traversalMethod_hasNot() != null + || next.traversalMethod_is() != null + || next.traversalMethod_where() != null + || next.traversalMethod_not() != null + || next.traversalMethod_identity() != null + || next.traversalMethod_limit() != null + || next.traversalMethod_order() != null + || next.traversalMethod_dedup() != null) { + // continue + } else { + break; + } + } + return tags.isEmpty() ? null : tags.get(tags.size() - 1); + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java index a23b30dee966..a09a1c1967f7 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java @@ -23,10 +23,7 @@ import com.alibaba.graphscope.gremlin.antlr4.GenericLiteralVisitor; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import org.antlr.v4.runtime.tree.ParseTree; -import org.checkerframework.checker.nullness.qual.Nullable; -import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -50,7 +47,8 @@ public PathExpandConfig.Builder visitTraversalMethod_out( int lower = Integer.valueOf(ranges[0]); int upper = Integer.valueOf(ranges[1]); // set path_opt and result_opt - List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); + List nextWith = + getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); nextWith.forEach(k -> visitTraversalMethod_with(k)); // set expand config, getV config, range return builder.expand(new ExpandConfig(GraphOpt.Expand.OUT, getExpandLabelConfig(labels))) @@ -68,7 +66,8 @@ public PathExpandConfig.Builder visitTraversalMethod_in( int lower = Integer.valueOf(ranges[0]); int upper = Integer.valueOf(ranges[1]); // set path_opt and result_opt - List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); + List nextWith = + getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); nextWith.forEach(k -> visitTraversalMethod_with(k)); // set expand config, getV config, range return builder.expand(new ExpandConfig(GraphOpt.Expand.IN, getExpandLabelConfig(labels))) @@ -86,7 +85,8 @@ public PathExpandConfig.Builder visitTraversalMethod_both( int lower = Integer.valueOf(ranges[0]); int upper = Integer.valueOf(ranges[1]); // set path_opt and result_opt - List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); + List nextWith = + getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); nextWith.forEach(k -> visitTraversalMethod_with(k)); // set expand config, getV config, range return builder.expand(new ExpandConfig(GraphOpt.Expand.BOTH, getExpandLabelConfig(labels))) @@ -98,21 +98,25 @@ public PathExpandConfig.Builder visitTraversalMethod_both( public PathExpandConfig.Builder visitTraversalMethod_with( GremlinGSParser.TraversalMethod_withContext ctx) { String optKey = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()); - Object optValue = GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral()); + Object optValue = + GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral()); switch (optKey.toUpperCase()) { case "PATH_OPT": - return builder.pathOpt(GraphOpt.PathExpandPath.valueOf(String.valueOf(optValue).toUpperCase())); + return builder.pathOpt( + GraphOpt.PathExpandPath.valueOf(String.valueOf(optValue).toUpperCase())); case "RESULT_OPT": - return builder.resultOpt(GraphOpt.PathExpandResult.valueOf(String.valueOf(optValue).toUpperCase())); + return builder.resultOpt( + GraphOpt.PathExpandResult.valueOf(String.valueOf(optValue).toUpperCase())); default: return builder; } } - private List getNextWith(GremlinGSParser.TraversalMethodContext curCtx) { + private List getNextWith( + GremlinGSParser.TraversalMethodContext curCtx) { List nextWith = Lists.newArrayList(); TraversalMethodIterator methodIterator = new TraversalMethodIterator(curCtx); - while(methodIterator.hasNext()) { + while (methodIterator.hasNext()) { GremlinGSParser.TraversalMethodContext next = methodIterator.next(); if (next.traversalMethod_with() != null) { nextWith.add(next.traversalMethod_with()); @@ -123,7 +127,6 @@ private List getNextWith(GremlinGSP return nextWith; } - private LabelConfig getExpandLabelConfig(String[] labels) { if (labels.length <= 1) { return new LabelConfig(true); @@ -136,33 +139,4 @@ private LabelConfig getExpandLabelConfig(String[] labels) { return expandLabels; } } - - private static class TraversalMethodIterator implements Iterator { - private GremlinGSParser.TraversalMethodContext _next; - public TraversalMethodIterator(GremlinGSParser.TraversalMethodContext start) { - this._next = start; - } - - @Override - public boolean hasNext() { - ParseTree parent = getParent(getParent(_next)); - return parent != null && parent.getChildCount() >= 3; - } - - @Override - public GremlinGSParser.TraversalMethodContext next() { - ParseTree parent = getParent(getParent(_next)); - if (parent == null || parent.getChildCount() < 3) return null; - _next = (GremlinGSParser.TraversalMethodContext) parent.getChild(2); - return _next; - } - - private @Nullable ParseTree getParent(ParseTree child) { - Class parentClass = GremlinGSParser.ChainedTraversalContext.class; - while (child != null && child.getParent() != null && !child.getParent().getClass().equals(parentClass)) { - child = child.getParent(); - } - return (child != null && child.getParent() != null && child.getParent().getClass().equals(parentClass)) ? child.getParent() : null; - } - } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/TraversalMethodIterator.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/TraversalMethodIterator.java new file mode 100644 index 000000000000..c48a2a5b1e8b --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/TraversalMethodIterator.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4x.visitor; + +import com.alibaba.graphscope.grammar.GremlinGSParser; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Iterator; +import java.util.Objects; + +public class TraversalMethodIterator implements Iterator { + private GremlinGSParser.TraversalMethodContext next; + private GremlinGSParser.TraversalSourceSpawnMethodContext sourceNext; + + public TraversalMethodIterator(GremlinGSParser.TraversalMethodContext next) { + this.next = Objects.requireNonNull(next); + } + + public TraversalMethodIterator(GremlinGSParser.TraversalSourceSpawnMethodContext sourceNext) { + this.sourceNext = Objects.requireNonNull(sourceNext); + this.next = null; + } + + @Override + public boolean hasNext() { + if (this.next == null) { + GremlinGSParser.RootTraversalContext rootCtx = + (GremlinGSParser.RootTraversalContext) sourceNext.getParent(); + return rootCtx != null && rootCtx.chainedTraversal() != null; + } + ParseTree parent = getParent(getParent(next)); + return parent != null && parent.getChildCount() >= 3; + } + + @Override + public GremlinGSParser.TraversalMethodContext next() { + if (this.next == null) { + GremlinGSParser.RootTraversalContext rootCtx = + (GremlinGSParser.RootTraversalContext) sourceNext.getParent(); + if (rootCtx == null || rootCtx.chainedTraversal() == null) return null; + next = leftDfs(rootCtx.chainedTraversal()); + return next; + } + ParseTree parent = getParent(getParent(next)); + if (parent == null || parent.getChildCount() < 3) return null; + next = (GremlinGSParser.TraversalMethodContext) parent.getChild(2); + return next; + } + + private @Nullable ParseTree getParent(ParseTree child) { + Class parentClass = GremlinGSParser.ChainedTraversalContext.class; + while (child != null + && child.getParent() != null + && !child.getParent().getClass().equals(parentClass)) { + child = child.getParent(); + } + return (child != null + && child.getParent() != null + && child.getParent().getClass().equals(parentClass)) + ? child.getParent() + : null; + } + + private GremlinGSParser.TraversalMethodContext leftDfs( + GremlinGSParser.ChainedTraversalContext root) { + if (root.chainedTraversal() != null) { + return leftDfs(root.chainedTraversal()); + } + return root.traversalMethod(); + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java index e7c42c2a2e7e..369fef982a62 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java @@ -18,9 +18,9 @@ import com.alibaba.graphscope.common.ir.Utils; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.gremlin.antlr4x.parser.GremlinAntlr4Parser; import com.alibaba.graphscope.gremlin.antlr4x.visitor.GraphBuilderVisitor; - import org.antlr.v4.runtime.tree.ParseTree; import org.apache.calcite.rel.RelNode; import org.junit.Test; @@ -43,8 +43,11 @@ public void g_V_out_test() { GraphBuilderVisitor visitor = new GraphBuilderVisitor(builder); ParseTree parseTree = new GremlinAntlr4Parser() - .parse("g.V().hasLabel('person').in('1..2', 'knows').with('result_opt', 'all_v').endV()"); + .parse("g.V().hasLabel('person').as('a').out().hasLabel('person').as('b').select('a', 'b').by('name').by()"); RelNode node = visitor.visit(parseTree).build(); + LogicalPlan plan = new LogicalPlan(node); System.out.println(node.explain()); + System.out.println(plan.getOutputType()); } + } From 06bdea752b867409547c96da6871081760ab02d1 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 27 Sep 2023 14:01:56 +0800 Subject: [PATCH 06/10] [GIE Compiler] ingegrate gremlin into calcite --- .../operator/SqlArrayValueConstructor.java | 2 +- .../rex/operator/SqlMapValueConstructor.java | 2 +- .../common/ir/tools/GraphBuilder.java | 10 +- .../ir/tools/GraphStdOperatorTable.java | 5 + .../common/ir/type/GraphSchemaType.java | 1 + .../antlr4x/visitor/ExpressionVisitor.java | 29 +- .../antlr4x/visitor/GraphBuilderVisitor.java | 92 +++- .../visitor/PathExpandBuilderVisitor.java | 27 +- .../script/AntlrGremlinScriptEngine.java | 4 +- .../sql/fun/ExtSqlPosixRegexOperator.java | 65 +++ .../gremlin/antlr4x/GraphBuilderTest.java | 428 +++++++++++++++++- 11 files changed, 606 insertions(+), 59 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/org/apache/calcite/sql/fun/ExtSqlPosixRegexOperator.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java index f571b62fe7dc..9d4e18ea81be 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlArrayValueConstructor.java @@ -55,7 +55,7 @@ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFail return (componentType == null) ? typeFactory.createSqlType(SqlTypeName.ANY) : componentType; - } catch (Exception e) { + } catch (AssertionError e) { return typeFactory.createSqlType(SqlTypeName.ANY); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java index 87fc5e5a2a48..e57103c4fc46 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rex/operator/SqlMapValueConstructor.java @@ -63,7 +63,7 @@ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFail return (componentType == null) ? typeFactory.createSqlType(SqlTypeName.ANY) : componentType; - } catch (Exception e) { + } catch (AssertionError e) { return typeFactory.createSqlType(SqlTypeName.ANY); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index 061c42ec6033..be83a0cc7b2a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -16,6 +16,8 @@ package com.alibaba.graphscope.common.ir.tools; +import static java.util.Objects.requireNonNull; + import com.alibaba.graphscope.common.ir.meta.schema.GraphOptSchema; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate; @@ -30,13 +32,14 @@ import com.alibaba.graphscope.common.ir.rel.type.group.GraphGroupKeys; import com.alibaba.graphscope.common.ir.rel.type.order.GraphFieldCollation; import com.alibaba.graphscope.common.ir.rel.type.order.GraphRelCollations; -import com.alibaba.graphscope.common.ir.rex.RexCallBinding; import com.alibaba.graphscope.common.ir.rex.*; +import com.alibaba.graphscope.common.ir.rex.RexCallBinding; import com.alibaba.graphscope.common.ir.tools.config.*; import com.alibaba.graphscope.common.ir.type.*; import com.alibaba.graphscope.gremlin.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.*; + import org.apache.calcite.plan.*; import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.RelFieldCollation; @@ -63,8 +66,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static java.util.Objects.requireNonNull; - /** * Integrate interfaces to build algebra structures, * including {@link RexNode} for expressions and {@link RelNode} for operators @@ -650,7 +651,8 @@ private boolean isCurrentSupported(SqlOperator operator) { || sqlKind == SqlKind.IS_NULL || sqlKind == SqlKind.IS_NOT_NULL || sqlKind == SqlKind.EXTRACT - || sqlKind == SqlKind.SEARCH; + || sqlKind == SqlKind.SEARCH + || sqlKind == SqlKind.POSIX_REGEX_CASE_SENSITIVE; } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java index 1b5402d39b6a..93906e12c463 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java @@ -22,6 +22,7 @@ import com.alibaba.graphscope.common.ir.rex.operator.SqlMapValueConstructor; import org.apache.calcite.sql.*; +import org.apache.calcite.sql.fun.ExtSqlPosixRegexOperator; import org.apache.calcite.sql.fun.SqlMonotonicBinaryOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.*; @@ -222,4 +223,8 @@ public static final SqlFunction USER_DEFINED_PROCEDURE(StoredProcedureMeta meta) SqlFunctionCategory.SYSTEM); public static final SqlOperator MAP_VALUE_CONSTRUCTOR = new SqlMapValueConstructor(); + + public static final SqlOperator POSIX_REGEX_CASE_SENSITIVE = + new ExtSqlPosixRegexOperator( + "POSIX REGEX CASE SENSITIVE", SqlKind.POSIX_REGEX_CASE_SENSITIVE, true, false); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java index c075a48377cc..acd4e33df0c2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphSchemaType.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; + import org.apache.calcite.linq4j.Ord; import org.apache.calcite.rel.type.*; import org.apache.commons.lang3.ObjectUtils; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java index 64b1f64719b6..5ff8402684d1 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java @@ -201,54 +201,65 @@ public RexNode visitTraversalPredicate_outside( @Override public RexNode visitTraversalPredicate_startingWith( GremlinGSParser.TraversalPredicate_startingWithContext ctx) { - String posixRegex = "^" + ctx.getText(); + String posixRegex = "^" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()); return builder.call( - GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, builder.literal(posixRegex)); + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + propertyKey, + builder.literal(posixRegex)); } @Override public RexNode visitTraversalPredicate_notStartingWith( GremlinGSParser.TraversalPredicate_notStartingWithContext ctx) { - String posixRegex = "^" + ctx.getText(); + String posixRegex = "^" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()); return builder.not( builder.call( GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + propertyKey, builder.literal(posixRegex))); } @Override public RexNode visitTraversalPredicate_endingWith( GremlinGSParser.TraversalPredicate_endingWithContext ctx) { - String posixRegex = ctx.getText() + "$"; + String posixRegex = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "$"; return builder.call( - GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, builder.literal(posixRegex)); + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + propertyKey, + builder.literal(posixRegex)); } @Override public RexNode visitTraversalPredicate_notEndingWith( GremlinGSParser.TraversalPredicate_notEndingWithContext ctx) { - String posixRegex = ctx.getText() + "$"; + String posixRegex = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "$"; return builder.not( builder.call( GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + propertyKey, builder.literal(posixRegex))); } @Override public RexNode visitTraversalPredicate_containing( GremlinGSParser.TraversalPredicate_containingContext ctx) { - String posixRegex = ".*" + ctx.getText() + "*."; + String posixRegex = + ".*" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "*."; return builder.call( - GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, builder.literal(posixRegex)); + GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + propertyKey, + builder.literal(posixRegex)); } @Override public RexNode visitTraversalPredicate_notContaining( GremlinGSParser.TraversalPredicate_notContainingContext ctx) { - String posixRegex = ".*" + ctx.getText() + "*."; + String posixRegex = + ".*" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "*."; return builder.not( builder.call( GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, + propertyKey, builder.literal(posixRegex))); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java index 6b5c2affb437..16128fa91b16 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/GraphBuilderVisitor.java @@ -59,7 +59,7 @@ public GraphBuilderVisitor(GraphBuilder builder) { @Override public GraphBuilder visitTraversalSourceSpawnMethod_V( GremlinGSParser.TraversalSourceSpawnMethod_VContext ctx) { - return builder.source( + builder.source( new SourceConfig( GraphOpt.Source.VERTEX, new LabelConfig(true), @@ -67,12 +67,31 @@ public GraphBuilder visitTraversalSourceSpawnMethod_V( new TraversalMethodIterator( (GremlinGSParser.TraversalSourceSpawnMethodContext) ctx.getParent())))); + if (ctx.integerLiteralList() != null) { + Object[] ids = GenericLiteralVisitor.getIntegerLiteralList(ctx.integerLiteralList()); + if (ids.length == 1) { + return builder.filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.ID_KEY), + builder.literal(ids[0]))); + } else if (ids.length > 1) { + List literals = + Arrays.asList(ids).stream() + .map(k -> builder.literal(k)) + .collect(Collectors.toList()); + return builder.filter( + builder.getRexBuilder() + .makeIn(builder.variable(null, GraphProperty.ID_KEY), literals)); + } + } + return builder; } @Override public GraphBuilder visitTraversalSourceSpawnMethod_E( GremlinGSParser.TraversalSourceSpawnMethod_EContext ctx) { - return builder.source( + builder.source( new SourceConfig( GraphOpt.Source.EDGE, new LabelConfig(true), @@ -80,6 +99,25 @@ public GraphBuilder visitTraversalSourceSpawnMethod_E( new TraversalMethodIterator( (GremlinGSParser.TraversalSourceSpawnMethodContext) ctx.getParent())))); + if (ctx.integerLiteralList() != null) { + Object[] ids = GenericLiteralVisitor.getIntegerLiteralList(ctx.integerLiteralList()); + if (ids.length == 1) { + return builder.filter( + builder.call( + GraphStdOperatorTable.EQUALS, + builder.variable(null, GraphProperty.ID_KEY), + builder.literal(ids[0]))); + } else if (ids.length > 1) { + List literals = + Arrays.asList(ids).stream() + .map(k -> builder.literal(k)) + .collect(Collectors.toList()); + return builder.filter( + builder.getRexBuilder() + .makeIn(builder.variable(null, GraphProperty.ID_KEY), literals)); + } + } + return builder; } @Override @@ -393,11 +431,14 @@ public GraphBuilder visitTraversalMethod_valueMap( builder.literal(k), builder.variable(null, k))) .collect(Collectors.toList())); - String tag = getNextTag( - new TraversalMethodIterator( - (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + String tag = + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); return builder.project( - ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); + ImmutableList.of(expr), + tag == null ? ImmutableList.of() : ImmutableList.of(tag), + true); } @Override @@ -407,11 +448,14 @@ public GraphBuilder visitTraversalMethod_values( RexNode expr = builder.variable( null, GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); - String tag = getNextTag( - new TraversalMethodIterator( - (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + String tag = + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); return builder.project( - ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); + ImmutableList.of(expr), + tag == null ? ImmutableList.of() : ImmutableList.of(tag), + true); } throw new UnsupportedEvalException(ctx.getClass(), "supported pattern is [values('..')]"); } @@ -429,19 +473,23 @@ public GraphBuilder visitTraversalMethod_elementMap( builder.literal(k), builder.variable(null, k))) .collect(Collectors.toList())); - String tag = getNextTag( - new TraversalMethodIterator( - (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + String tag = + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); return builder.project( - ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); + ImmutableList.of(expr), + tag == null ? ImmutableList.of() : ImmutableList.of(tag), + true); } @Override public GraphBuilder visitTraversalMethod_select( GremlinGSParser.TraversalMethod_selectContext ctx) { - String tag = getNextTag( - new TraversalMethodIterator( - (GremlinGSParser.TraversalMethodContext) ctx.getParent())); + String tag = + getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); RexNode expr; if (ctx.stringLiteral() != null) { List selectTags = @@ -474,7 +522,7 @@ public GraphBuilder visitTraversalMethod_select( mapParameters.add(builder.literal(k)); mapParameters.add(v); }); - expr = builder.call(GraphStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, mapParameters); + expr = builder.call(GraphStdOperatorTable.MAP_VALUE_CONSTRUCTOR, mapParameters); } } else if (ctx.traversalColumn() != null) { Column column = @@ -486,7 +534,10 @@ public GraphBuilder visitTraversalMethod_select( GremlinGSParser.TraversalMethod_selectbyContext.class, ctx.getText() + " is unsupported yet"); } - return builder.project(ImmutableList.of(expr), tag == null ? ImmutableList.of() : ImmutableList.of(tag), true); + return builder.project( + ImmutableList.of(expr), + tag == null ? ImmutableList.of() : ImmutableList.of(tag), + true); } public GraphBuilder getGraphBuilder() { @@ -600,7 +651,8 @@ private List getAllProperties(@Nullable String tag) { || next.traversalMethod_identity() != null || next.traversalMethod_limit() != null || next.traversalMethod_order() != null - || next.traversalMethod_dedup() != null) { + || next.traversalMethod_dedup() != null + || next.traversalMethod_with() != null) { // continue } else { break; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java index a09a1c1967f7..aec29f295d64 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/PathExpandBuilderVisitor.java @@ -50,10 +50,15 @@ public PathExpandConfig.Builder visitTraversalMethod_out( List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); nextWith.forEach(k -> visitTraversalMethod_with(k)); - // set expand config, getV config, range + // set expand config, getV config, range, alias + String alias = + parent.getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); return builder.expand(new ExpandConfig(GraphOpt.Expand.OUT, getExpandLabelConfig(labels))) .getV(new GetVConfig(GraphOpt.GetV.END)) - .range(lower, upper - lower); + .range(lower, upper - lower) + .alias(alias); } @Override @@ -69,10 +74,15 @@ public PathExpandConfig.Builder visitTraversalMethod_in( List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); nextWith.forEach(k -> visitTraversalMethod_with(k)); - // set expand config, getV config, range + // set expand config, getV config, range, alias + String alias = + parent.getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); return builder.expand(new ExpandConfig(GraphOpt.Expand.IN, getExpandLabelConfig(labels))) .getV(new GetVConfig(GraphOpt.GetV.START)) - .range(lower, upper - lower); + .range(lower, upper - lower) + .alias(alias); } @Override @@ -88,10 +98,15 @@ public PathExpandConfig.Builder visitTraversalMethod_both( List nextWith = getNextWith((GremlinGSParser.TraversalMethodContext) ctx.getParent()); nextWith.forEach(k -> visitTraversalMethod_with(k)); - // set expand config, getV config, range + // set expand config, getV config, range, alias + String alias = + parent.getNextTag( + new TraversalMethodIterator( + (GremlinGSParser.TraversalMethodContext) ctx.getParent())); return builder.expand(new ExpandConfig(GraphOpt.Expand.BOTH, getExpandLabelConfig(labels))) .getV(new GetVConfig(GraphOpt.GetV.OTHER)) - .range(lower, upper - lower); + .range(lower, upper - lower) + .alias(alias); } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java index 2b2ecf543f3c..259f085d1a59 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java @@ -29,6 +29,7 @@ import com.alibaba.graphscope.grammar.GremlinGSLexer; import com.alibaba.graphscope.grammar.GremlinGSParser; import com.alibaba.graphscope.gremlin.antlr4.GremlinAntlrToJava; + import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.DefaultErrorStrategy; @@ -42,11 +43,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Reader; + import javax.script.AbstractScriptEngine; import javax.script.Bindings; import javax.script.ScriptContext; import javax.script.SimpleBindings; -import java.io.Reader; public class AntlrGremlinScriptEngine extends AbstractScriptEngine implements GremlinScriptEngine { private Logger logger = LoggerFactory.getLogger(AntlrGremlinScriptEngine.class); diff --git a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/fun/ExtSqlPosixRegexOperator.java b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/fun/ExtSqlPosixRegexOperator.java new file mode 100644 index 000000000000..69806b258a99 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/fun/ExtSqlPosixRegexOperator.java @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.calcite.sql.fun; + +import com.alibaba.graphscope.common.ir.rex.RexCallBinding; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.util.Static; +import org.apache.calcite.util.Util; + +public class ExtSqlPosixRegexOperator extends SqlPosixRegexOperator { + public ExtSqlPosixRegexOperator( + String name, SqlKind kind, boolean caseSensitive, boolean negated) { + super(name, kind, caseSensitive, negated); + } + + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + int operandCount = callBinding.getOperandCount(); + if (operandCount != 2) { + throw new AssertionError( + "Unexpected number of args to " + callBinding.getCall() + ": " + operandCount); + } else { + RelDataType op1Type = callBinding.getOperandType(0); + RelDataType op2Type = callBinding.getOperandType(1); + if (!SqlTypeUtil.isComparable(op1Type, op2Type)) { + throw new AssertionError( + "Incompatible first two operand types " + op1Type + " and " + op2Type); + } else { + if (!SqlTypeUtil.isCharTypeComparable(callBinding.collectOperandTypes())) { + if (throwOnFailure) { + String msg = + String.join( + ", ", + Util.transform( + ((RexCallBinding) callBinding).getRexOperands(), + String::valueOf)); + throw callBinding.newError(Static.RESOURCE.operandNotComparable(msg)); + } else { + return false; + } + } else { + return true; + } + } + } + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java index 369fef982a62..d49960742de6 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java @@ -18,36 +18,430 @@ import com.alibaba.graphscope.common.ir.Utils; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.gremlin.antlr4x.parser.GremlinAntlr4Parser; import com.alibaba.graphscope.gremlin.antlr4x.visitor.GraphBuilderVisitor; + import org.antlr.v4.runtime.tree.ParseTree; import org.apache.calcite.rel.RelNode; +import org.junit.Assert; import org.junit.Test; public class GraphBuilderTest { - @Test - public void g_V_test() { + public static RelNode eval(String query) { GraphBuilder builder = Utils.mockGraphBuilder(); GraphBuilderVisitor visitor = new GraphBuilderVisitor(builder); - ParseTree parseTree = - new GremlinAntlr4Parser() - .parse("g.V().hasLabel('person').hasId(1, 2, 3).hasNot('name')"); - RelNode node = visitor.visit(parseTree).build(); - System.out.println(node.explain()); + ParseTree parseTree = new GremlinAntlr4Parser().parse(query); + return visitor.visit(parseTree).build(); + } + + @Test + public void g_V_test() { + RelNode node = eval("g.V()"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_id_test() { + RelNode node = eval("g.V(1, 2, 3)"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX], uniqueKeyFilters=[SEARCH(DEFAULT.~id," + + " Sarg[1, 2, 3])])", + node.explain().trim()); + } + + @Test + public void g_V_hasLabel_test() { + RelNode node = eval("g.V().hasLabel('person')"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}], alias=[DEFAULT]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_hasLabel_as_test() { + RelNode node = eval("g.V().hasLabel('person').as('a')"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}], alias=[a]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_hasLabels_as_test() { + RelNode node = eval("g.V().hasLabel('person', 'software').as('a')"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[a], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_hasLabel_error_test() { + try { + RelNode node = eval("g.V().hasLabel('person').hasLabel('software')"); + } catch (IllegalArgumentException e) { + Assert.assertEquals( + "cannot find common labels between values= [software] and label=" + + " [[VertexLabel(person)]]", + e.getMessage()); + return; + } + Assert.fail(); + } + + @Test + public void g_V_hasId_test() { + RelNode node = eval("g.V().hasId(1)"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX], uniqueKeyFilters=[=(DEFAULT.~id, 1)])", + node.explain().trim()); + } + + @Test + public void g_V_hasIds_test() { + RelNode node = eval("g.V().hasId(1, 2, 3)"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX], uniqueKeyFilters=[SEARCH(DEFAULT.~id," + + " Sarg[1, 2, 3])])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_marko_test() { + RelNode node = eval("g.V().has('name', 'marko')"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[=(DEFAULT.name, _UTF-8'marko')]]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_eq_marko_test() { + RelNode node = eval("g.V().has('name', eq('marko'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[=(DEFAULT.name, _UTF-8'marko')]]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_neq_marko_test() { + RelNode node = eval("g.V().has('name', eq('marko'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[<>(DEFAULT.name, _UTF-8'marko')]]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_age_gt_17_test() { + RelNode node = eval("g.V().has('age', gt(17))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[>(DEFAULT.age, 17)]], opt=[VERTEX])", + node.explain().trim()); + } + + // todo: convert SEARCH operator to ir core structure + @Test + public void g_V_has_age_inside_17_20_test() { + RelNode node = eval("g.V().has('age', inside(17, 20))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[[18..19]])]]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_age_outside_17_20_test() { + RelNode node = eval("g.V().has('age', outside(17, 20))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[(-∞..18)," + + " (19..+∞)])]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_age_within_test() { + RelNode node = eval("g.V().has('age', within(17, 20))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[17, 20])]]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_age_without_test() { + RelNode node = eval("g.V().has('age', without(17, 20))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[(-∞..17), (17..20)," + + " (20..+∞)])]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_endingWith_test() { + RelNode node = eval("g.V().has('name', endingWith('mar'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[POSIX REGEX CASE SENSITIVE(DEFAULT.name," + + " _UTF-8'mar$')]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_not_endingWith_test() { + RelNode node = eval("g.V().has('name', notEndingWith('mar'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[NOT(POSIX REGEX CASE SENSITIVE(DEFAULT.name," + + " _UTF-8'mar$'))]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_containing_test() { + RelNode node = eval("g.V().has('name', containing('mar'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[POSIX REGEX CASE SENSITIVE(DEFAULT.name," + + " _UTF-8'.*mar*.')]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_not_containing_test() { + RelNode node = eval("g.V().has('name', notContaining('mar'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[NOT(POSIX REGEX CASE SENSITIVE(DEFAULT.name," + + " _UTF-8'.*mar*.'))]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_startingWith_test() { + RelNode node = eval("g.V().has('name', startingWith('mar'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[POSIX REGEX CASE SENSITIVE(DEFAULT.name," + + " _UTF-8'^mar')]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_not_startingWith_test() { + RelNode node = eval("g.V().has('name', notStartingWith('mar'))"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[NOT(POSIX REGEX CASE SENSITIVE(DEFAULT.name," + + " _UTF-8'^mar'))]], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_marko_age_17_has_label_test() { + RelNode node = eval("g.V().has('name', 'marko').has('age', 17).hasLabel('person')"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}], alias=[DEFAULT]," + + " fusedFilter=[[AND(=(DEFAULT.name, _UTF-8'marko'), =(DEFAULT.age, 17))]]," + + " opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_has_name_marko_age_17_has_label_error_test() { + try { + RelNode node = eval("g.V().has('name', 'marko').has('age', 17).hasLabel('software')"); + } catch (IllegalArgumentException e) { + Assert.assertEquals( + "{property=age} not found; expected properties are: [id, name, lang," + + " creationDate]", + e.getMessage()); + return; + } + Assert.fail(); } @Test public void g_V_out_test() { - GraphBuilder builder = Utils.mockGraphBuilder(); - GraphBuilderVisitor visitor = new GraphBuilderVisitor(builder); - ParseTree parseTree = - new GremlinAntlr4Parser() - .parse("g.V().hasLabel('person').as('a').out().hasLabel('person').as('b').select('a', 'b').by('name').by()"); - RelNode node = visitor.visit(parseTree).build(); - LogicalPlan plan = new LogicalPlan(node); - System.out.println(node.explain()); - System.out.println(plan.getOutputType()); + RelNode node = eval("g.V().out()"); + Assert.assertEquals( + "GraphLogicalGetV(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[END])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=true, tables=[created, knows]}]," + + " alias=[DEFAULT], opt=[OUT])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software," + + " person]}], alias=[DEFAULT], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_out_label_test() { + RelNode node = eval("g.V().out('knows')"); + Assert.assertEquals( + "GraphLogicalGetV(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[END])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[knows]}]," + + " alias=[DEFAULT], opt=[OUT])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software," + + " person]}], alias=[DEFAULT], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_out_label_as_test() { + RelNode node = eval("g.V().as('a').out().as('b')"); + Assert.assertEquals( + "GraphLogicalGetV(tableConfig=[{isAll=true, tables=[software, person]}], alias=[b]," + + " opt=[END])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=true, tables=[created, knows]}]," + + " alias=[DEFAULT], opt=[OUT])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software," + + " person]}], alias=[a], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_path_expand_test() { + RelNode node = + eval( + "g.V().as('a').out('1..2', 'knows').with('path_opt'," + + " 'simple').with('result_opt', 'all_v').as('b').endV().as('c')"); + Assert.assertEquals( + "GraphLogicalGetV(tableConfig=[{isAll=true, tables=[software, person]}], alias=[c]," + + " opt=[END])\n" + + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," + + " tables=[knows]}], alias=[DEFAULT], opt=[OUT])\n" + + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=true, tables=[software," + + " person]}], alias=[DEFAULT], opt=[END])\n" + + "], offset=[1], fetch=[1], path_opt=[SIMPLE], result_opt=[ALL_V]," + + " alias=[b])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software," + + " person]}], alias=[a], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_values_name_test() { + RelNode node = eval("g.V().values('name').as('a')"); + Assert.assertEquals( + "GraphLogicalProject(a=[DEFAULT.name], isAppend=[true])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_valueMap_name_test() { + RelNode node = eval("g.V().valueMap('name').as('a')"); + Assert.assertEquals( + "GraphLogicalProject(a=[MAP(_UTF-8'name', DEFAULT.name)], isAppend=[true])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_valueMap_all_test() { + RelNode node = eval("g.V().valueMap().as('a')"); + Assert.assertEquals( + "GraphLogicalProject(a=[MAP(_UTF-8'id', DEFAULT.id, _UTF-8'name', DEFAULT.name," + + " _UTF-8'lang', DEFAULT.lang, _UTF-8'creationDate', DEFAULT.creationDate," + + " _UTF-8'age', DEFAULT.age)], isAppend=[true])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_elementMap_all_test() { + RelNode node = eval("g.V().elementMap().as('a')"); + Assert.assertEquals( + "GraphLogicalProject(a=[MAP(_UTF-8'~label', DEFAULT.~label, _UTF-8'~id'," + + " DEFAULT.~id, _UTF-8'id', DEFAULT.id, _UTF-8'name', DEFAULT.name," + + " _UTF-8'lang', DEFAULT.lang, _UTF-8'creationDate', DEFAULT.creationDate," + + " _UTF-8'age', DEFAULT.age)], isAppend=[true])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], opt=[VERTEX])", + node.explain().trim()); } + // todo: optimize select('a'), provide a tag representing 'NONE' + @Test + public void g_V_select_a_test() { + RelNode node = eval("g.V().as('a').select('a')"); + Assert.assertEquals( + "GraphLogicalProject(a0=[a], isAppend=[true])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[a], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_select_a_by_name_test() { + RelNode node = eval("g.V().as('a').select('a').by('name')"); + Assert.assertEquals( + "GraphLogicalProject(name=[a.name], isAppend=[true])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[a], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_select_a_by_valueMap_test() { + RelNode node = eval("g.V().as('a').select('a').by(valueMap())"); + Assert.assertEquals( + "GraphLogicalProject($f0=[MAP(_UTF-8'id', a.id, _UTF-8'name', a.name, _UTF-8'lang'," + + " a.lang, _UTF-8'creationDate', a.creationDate, _UTF-8'age', a.age)]," + + " isAppend=[true])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[a], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_select_a_b_test() { + RelNode node = eval("g.V().as('a').out().as('b').select('a', 'b')"); + Assert.assertEquals( + "GraphLogicalProject($f0=[MAP(_UTF-8'a', a, _UTF-8'b', b)], isAppend=[true])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[b], opt=[END])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=true, tables=[created, knows]}]," + + " alias=[DEFAULT], opt=[OUT])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software," + + " person]}], alias=[a], opt=[VERTEX])", + node.explain().trim()); + } + + @Test + public void g_V_select_a_b_by_name_valueMap_test() { + RelNode node = + eval("g.V().as('a').out().as('b').select('a', 'b').by('name').by(valueMap())"); + Assert.assertEquals( + "GraphLogicalProject($f0=[MAP(_UTF-8'a', a.name, _UTF-8'b', MAP(_UTF-8'id', b.id," + + " _UTF-8'name', b.name, _UTF-8'lang', b.lang, _UTF-8'creationDate'," + + " b.creationDate, _UTF-8'age', b.age))], isAppend=[true])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[b], opt=[END])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=true, tables=[created, knows]}]," + + " alias=[DEFAULT], opt=[OUT])\n" + + " GraphLogicalSource(tableConfig=[{isAll=true, tables=[software," + + " person]}], alias=[a], opt=[VERTEX])", + node.explain().trim()); + } } From f80c160d241ef724b816fca20cf5a1c62cdfd134 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 27 Sep 2023 14:08:38 +0800 Subject: [PATCH 07/10] [GIE Compiler] revert changes --- .../script/AntlrGremlinScriptEngine.java | 9 +- .../graphscope/common/ir/ExpressionTest.java | 123 +----------------- 2 files changed, 5 insertions(+), 127 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java index 259f085d1a59..dbf8ff49f84d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrGremlinScriptEngine.java @@ -30,9 +30,7 @@ import com.alibaba.graphscope.grammar.GremlinGSParser; import com.alibaba.graphscope.gremlin.antlr4.GremlinAntlrToJava; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.atn.PredictionMode; import org.apache.commons.lang3.NotImplementedException; import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; @@ -45,10 +43,7 @@ import java.io.Reader; -import javax.script.AbstractScriptEngine; -import javax.script.Bindings; -import javax.script.ScriptContext; -import javax.script.SimpleBindings; +import javax.script.*; public class AntlrGremlinScriptEngine extends AbstractScriptEngine implements GremlinScriptEngine { private Logger logger = LoggerFactory.getLogger(AntlrGremlinScriptEngine.class); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java index d6887455db27..e64f23570e52 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java @@ -18,11 +18,10 @@ import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; -import com.alibaba.graphscope.common.ir.tools.config.*; -import com.alibaba.graphscope.common.ir.type.GraphProperty; -import com.google.common.collect.ImmutableList; +import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.alibaba.graphscope.common.ir.tools.config.LabelConfig; +import com.alibaba.graphscope.common.ir.tools.config.SourceConfig; -import org.apache.calcite.rel.RelNode; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.SqlTypeName; import org.junit.Assert; @@ -170,124 +169,8 @@ public void unary_minus_test() { Assert.assertEquals("-(a.age)", node.toString()); } - @Test - public void label_1_test() { - RelNode node = - builder.source(new SourceConfig(GraphOpt.Source.VERTEX, new LabelConfig(true), "a")) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.literal("software"), - builder.variable(null, GraphProperty.LABEL_KEY))) - .build(); - Assert.assertEquals( - "GraphLogicalSource(tableConfig=[{isAll=false, tables=[software]}], alias=[a]," - + " opt=[VERTEX])", - node.explain().trim()); - } - - // check property after updating the label type - @Test - public void label_2_test() { - try { - RelNode node = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, new LabelConfig(true), "a")) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable(null, "age"), - builder.literal(1))) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.literal("software"), - builder.variable(null, GraphProperty.LABEL_KEY))) - .build(); - } catch (IllegalArgumentException e) { - Assert.assertEquals( - "{property=age} not found; expected properties are: [id, name, lang," - + " creationDate]", - e.getMessage()); - return; - } - Assert.fail(); - } - - @Test - public void label_3_test() { - RelNode node = - builder.source(new SourceConfig(GraphOpt.Source.VERTEX, new LabelConfig(true), "a")) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable(null, GraphProperty.ID_KEY), - builder.literal(1))) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.literal("software"), - builder.variable(null, GraphProperty.LABEL_KEY))) - .filter( - builder.call( - GraphStdOperatorTable.AND, - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable(null, "~id"), - builder.literal(3)), - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable(null, GraphProperty.ID_KEY), - builder.literal(2)))) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable(null, "lang"), - builder.literal("a"))) - .build(); - System.out.println(node.explain().trim()); - } - private SourceConfig mockSourceConfig(String alias) { return new SourceConfig( GraphOpt.Source.VERTEX, new LabelConfig(false).addLabel("person"), alias); } - - // @Test - // public void label_2_test() { - // RexNode var = - // builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) - // .variable(null, GraphProperty.LABEL_KEY); - // RexNode inNode = - // builder.getRexBuilder().makeIn(var, - // ImmutableList.of(builder.literal("person"))); - // RelNode node = - // builder.filter(inNode) - // .filter( - // builder.getRexBuilder() - // .makeIn(var, - // ImmutableList.of(builder.literal("software")))) - // .build(); - // System.out.println(node.explain()); - // } - - @Test - public void label_4_test() { - RexNode var = - builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) - .variable(null, GraphProperty.ID_KEY); - RexNode inNode = - builder.getRexBuilder() - .makeIn(var, ImmutableList.of(builder.literal(1), builder.literal(2))); - RelNode node = - builder.filter(inNode) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable(null, "name"), - builder.literal("marko"))) - .build(); - System.out.println(node.explain()); - } } From 87ee2ea505f2863876fd78493e420ea1c7a25074 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 27 Sep 2023 14:38:48 +0800 Subject: [PATCH 08/10] [GIE Compiler] minor fix --- .../graphscope/cypher/antlr4/WithTest.java | 3 +-- .../gremlin/antlr4x/GraphBuilderTest.java | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java index 6eb36f2acf51..9076848fdd25 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java @@ -211,8 +211,7 @@ public void with_12_test() { RelNode project = Utils.eval("Match (a:person)-[]-(b:person) Return [a.name, b.age, 1]").build(); Assert.assertEquals( - "GraphLogicalProject($f0=[ARRAY_VALUE_CONSTRUCTOR(a.name, b.age, 1)]," - + " isAppend=[false])\n" + "GraphLogicalProject($f0=[ARRAY(a.name, b.age, 1)], isAppend=[false])\n" + " GraphLogicalSingleMatch(input=[null]," + " sentence=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," + " alias=[b], opt=[OTHER])\n" diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java index d49960742de6..6e718fd1fa8b 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java @@ -135,7 +135,7 @@ public void g_V_has_name_eq_marko_test() { @Test public void g_V_has_name_neq_marko_test() { - RelNode node = eval("g.V().has('name', eq('marko'))"); + RelNode node = eval("g.V().has('name', neq('marko'))"); Assert.assertEquals( "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + " alias=[DEFAULT], fusedFilter=[[<>(DEFAULT.name, _UTF-8'marko')]]," @@ -152,6 +152,16 @@ public void g_V_has_age_gt_17_test() { node.explain().trim()); } + // hasNot('age') -> age is null + @Test + public void g_V_hasNot_age_test() { + RelNode node = eval("g.V().hasNot('age')"); + Assert.assertEquals( + "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," + + " alias=[DEFAULT], fusedFilter=[[IS NULL(DEFAULT.age)]], opt=[VERTEX])", + node.explain().trim()); + } + // todo: convert SEARCH operator to ir core structure @Test public void g_V_has_age_inside_17_20_test() { @@ -178,8 +188,8 @@ public void g_V_has_age_within_test() { RelNode node = eval("g.V().has('age', within(17, 20))"); Assert.assertEquals( "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," - + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[17, 20])]]," - + " opt=[VERTEX])", + + " alias=[DEFAULT], fusedFilter=[[SEARCH(DEFAULT.age, Sarg[17, 20])]]," + + " opt=[VERTEX])", node.explain().trim()); } From d9b37f59485223d5c20c63f5ed355a4e514a00e7 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 11 Oct 2023 19:19:49 +0800 Subject: [PATCH 09/10] [GIE Compiler] convert index predicate from RexNode expression to ir core --- .../ir/rel/graph/GraphLogicalSource.java | 9 -- .../ir/runtime/ffi/FfiPhysicalBuilder.java | 3 +- .../ir/runtime/ffi/RelToFfiConverter.java | 114 ++++++++++++------ .../common/ir/tools/GraphBuilder.java | 51 +++++--- .../common/ir/tools/GraphPlanner.java | 7 +- 5 files changed, 119 insertions(+), 65 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java index 5d5b3bbb1c94..2d6fce56a41c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java @@ -31,7 +31,6 @@ public class GraphLogicalSource extends AbstractBindableTableScan { private final GraphOpt.Source opt; private @Nullable RexNode uniqueKeyFilters; - private @Nullable RexNode primaryKeyFilters; protected GraphLogicalSource( GraphOptCluster cluster, @@ -67,15 +66,7 @@ public void setUniqueKeyFilters(RexNode uniqueKeyFilters) { this.uniqueKeyFilters = Objects.requireNonNull(uniqueKeyFilters); } - public void setPrimaryKeyFilters(RexNode primaryKeyFilters) { - this.primaryKeyFilters = Objects.requireNonNull(primaryKeyFilters); - } - public @Nullable RexNode getUniqueKeyFilters() { return uniqueKeyFilters; } - - public @Nullable RexNode getPrimaryKeyFilters() { - return primaryKeyFilters; - } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java index 9216217d96da..b05ce9b46651 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java @@ -68,7 +68,8 @@ public FfiPhysicalBuilder( Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan, PlanPointer planPointer) { super( logicalPlan, - new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId()))); + new GraphRelShuttleWrapper( + new RelToFfiConverter(irMeta.getSchema().isColumnId(), graphConfig))); this.graphConfig = graphConfig; this.irMeta = irMeta; this.planPointer = Objects.requireNonNull(planPointer); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java index f9efd30c80c9..fb044d9e4402 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.common.ir.runtime.ffi; +import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.intermediate.ArgUtils; import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate; import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject; @@ -31,8 +32,10 @@ import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter; import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode; import com.alibaba.graphscope.common.ir.tools.AliasInference; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.alibaba.graphscope.common.ir.type.GraphNameOrId; import com.alibaba.graphscope.common.ir.type.GraphProperty; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.common.jna.IrCoreLibrary; @@ -49,17 +52,13 @@ import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexVariable; +import org.apache.calcite.rex.*; import org.apache.calcite.sql.SqlKind; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -72,9 +71,11 @@ public class RelToFfiConverter implements GraphRelShuttle { private static final Logger logger = LoggerFactory.getLogger(RelToFfiConverter.class); private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE; private final boolean isColumnId; + private final RexBuilder rexBuilder; - public RelToFfiConverter(boolean isColumnId) { + public RelToFfiConverter(boolean isColumnId, Configs configs) { this.isColumnId = isColumnId; + this.rexBuilder = GraphPlanner.rexBuilderFactory.apply(configs); } @Override @@ -460,46 +461,85 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) { } private @Nullable Pointer ffiIndexPredicates(GraphLogicalSource source) { - ImmutableList filters = source.getFilters(); - if (ObjectUtils.isEmpty(filters)) return null; - // decomposed by OR - List disJunctions = RelOptUtil.disjunctions(filters.get(0)); - List literals = new ArrayList<>(); - for (RexNode rexNode : disJunctions) { - if (!isIdEqualsLiteral(rexNode, literals)) { - return null; - } - } + RexNode uniqueKeyFilters = source.getUniqueKeyFilters(); + if (uniqueKeyFilters == null) return null; + // 'within' operator in index predicate is unsupported in ir core, here just expand it to + // 'or' + // i.e. '~id within [1, 2]' -> '~id == 1 or ~id == 2' + RexNode expandSearch = RexUtil.expandSearch(this.rexBuilder, null, uniqueKeyFilters); + List disjunctions = RelOptUtil.disjunctions(expandSearch); Pointer ptrIndex = LIB.initIndexPredicate(); - for (RexNode literal : literals) { - FfiProperty.ByValue property = new FfiProperty.ByValue(); - property.opt = FfiPropertyOpt.Id; - checkFfiResult( - LIB.orEquivPredicate(ptrIndex, property, Utils.ffiConst((RexLiteral) literal))); + for (RexNode disjunction : disjunctions) { + if (disjunction instanceof RexCall) { + RexCall rexCall = (RexCall) disjunction; + switch (rexCall.getOperator().getKind()) { + case EQUALS: + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (left instanceof RexGraphVariable + && (right instanceof RexLiteral + || right instanceof RexDynamicParam)) { + LIB.orEquivPredicate( + ptrIndex, + getFfiProperty(((RexGraphVariable) left).getProperty()), + getFfiConst(right)); + break; + } else if (right instanceof RexGraphVariable + && (left instanceof RexLiteral + || left instanceof RexDynamicParam)) { + LIB.orEquivPredicate( + ptrIndex, + getFfiProperty(((RexGraphVariable) right).getProperty()), + getFfiConst(left)); + break; + } + default: + throw new IllegalArgumentException( + "can not convert unique key filter pattern=" + + rexCall + + " to ir core index predicate"); + } + } else { + throw new IllegalArgumentException( + "invalid unique key filter pattern=" + disjunction); + } } - // remove index predicates from filter conditions - source.setFilters(ImmutableList.of()); return ptrIndex; } - // i.e. a.~id == 10 - private boolean isIdEqualsLiteral(RexNode rexNode, List literal) { - if (rexNode.getKind() != SqlKind.EQUALS) return false; - List operands = ((RexCall) rexNode).getOperands(); - return isGlobalId(operands.get(0)) && isLiteral(operands.get(1), literal) - || isGlobalId(operands.get(1)) && isLiteral(operands.get(0), literal); + private FfiProperty.ByValue getFfiProperty(GraphProperty property) { + Preconditions.checkArgument(property != null, "unique key should not be null"); + FfiProperty.ByValue ffiProperty = new FfiProperty.ByValue(); + switch (property.getOpt()) { + case ID: + ffiProperty.opt = FfiPropertyOpt.Id; + break; + case KEY: + ffiProperty.opt = FfiPropertyOpt.Key; + ffiProperty.key = getFfiNameOrId(property.getKey()); + break; + default: + throw new IllegalArgumentException( + "can not convert property=" + property + " to ffi property"); + } + return ffiProperty; } - // i.e. a.~id - private boolean isGlobalId(RexNode rexNode) { - return rexNode instanceof RexGraphVariable - && ((RexGraphVariable) rexNode).getProperty().getOpt() == GraphProperty.Opt.ID; + private FfiNameOrId.ByValue getFfiNameOrId(GraphNameOrId nameOrId) { + switch (nameOrId.getOpt()) { + case NAME: + return ArgUtils.asNameOrId(nameOrId.getName()); + case ID: + default: + return ArgUtils.asNameOrId(nameOrId.getId()); + } } - private boolean isLiteral(RexNode rexNode, List literal) { - boolean isLiteral = rexNode.getKind() == SqlKind.LITERAL; - if (isLiteral) literal.add(rexNode); - return isLiteral; + private FfiConst.ByValue getFfiConst(RexNode rexNode) { + if (rexNode instanceof RexLiteral) { + return Utils.ffiConst((RexLiteral) rexNode); + } + throw new IllegalArgumentException("cannot convert rexNode=" + rexNode + " to ffi const"); } private List range(RexNode offset, RexNode fetch) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index be83a0cc7b2a..d917e58d0b59 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -769,9 +769,11 @@ public Void visitInputRef(RexInputRef inputRef) { } }; if (tableScan instanceof GraphLogicalSource) { - GraphLogicalSource source = (GraphLogicalSource) tableScan; - if (source.getUniqueKeyFilters() != null) { - builder.filter(source.getUniqueKeyFilters()); + RexNode originalUniqueKeyFilters = + ((GraphLogicalSource) tableScan).getUniqueKeyFilters(); + if (originalUniqueKeyFilters != null) { + originalUniqueKeyFilters.accept(propertyChecker); + builder.filter(originalUniqueKeyFilters); } } ImmutableList originalFilters = tableScan.getFilters(); @@ -788,12 +790,11 @@ public Void visitInputRef(RexInputRef inputRef) { } if (tableScan instanceof GraphLogicalSource && !uniqueKeyFilters.isEmpty()) { GraphLogicalSource source = (GraphLogicalSource) tableScan; - if (source.getUniqueKeyFilters() != null) { - uniqueKeyFilters.add(0, source.getUniqueKeyFilters()); - } - // todo: check if unique key filters have common keys, otherwise throw errors + Preconditions.checkArgument( + source.getUniqueKeyFilters() == null, + "can not add unique key filters if original is not empty"); source.setUniqueKeyFilters( - RexUtil.composeConjunction(this.getRexBuilder(), uniqueKeyFilters)); + RexUtil.composeDisjunction(this.getRexBuilder(), uniqueKeyFilters)); } if (!extraFilters.isEmpty()) { ImmutableList originalFilters = tableScan.getFilters(); @@ -813,7 +814,7 @@ private void classifyFilters( AbstractBindableTableScan tableScan, RexNode condition, List labelValues, - List uniqueKeyFilters, + List uniqueKeyFilters, // unique key filters int the list are composed by 'OR' List filters) { List conjunctions = RelOptUtil.conjunctions(condition); List filtersToRemove = Lists.newArrayList(); @@ -836,13 +837,26 @@ private void classifyFilters( getValuesAsList(((RexLiteral) left).getValueAs(Comparable.class))); break; } - if (tableScan instanceof GraphLogicalSource) { - if (isUniqueKey(left) && right instanceof RexLiteral) { - filtersToRemove.add(conjunction); - uniqueKeyFilters.add(conjunction); - } else if (left instanceof RexLiteral && isUniqueKey(right)) { - filtersToRemove.add(conjunction); - uniqueKeyFilters.add(conjunction); + } + } + } + if (tableScan instanceof GraphLogicalSource + && ((GraphLogicalSource) tableScan).getUniqueKeyFilters() == null) { + // try to extract unique key filters from the original condition + List disjunctions = RelOptUtil.disjunctions(condition); + for (RexNode disjunction : disjunctions) { + if (disjunction instanceof RexCall) { + RexCall rexCall = (RexCall) disjunction; + if (rexCall.getOperator().getKind() == SqlKind.EQUALS + || rexCall.getOperator().getKind() == SqlKind.SEARCH) { + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (isUniqueKey(left) && isLiteralOrDynamicParams(right)) { + filtersToRemove.add(disjunction); + uniqueKeyFilters.add(disjunction); + } else if (isLiteralOrDynamicParams(left) && isUniqueKey(right)) { + filtersToRemove.add(disjunction); + uniqueKeyFilters.add(disjunction); } } } @@ -855,6 +869,7 @@ private void classifyFilters( } private boolean isUniqueKey(RexNode rexNode) { + // todo: support primary keys if (rexNode instanceof RexGraphVariable) { RexGraphVariable variable = (RexGraphVariable) rexNode; return variable.getProperty() != null @@ -863,6 +878,10 @@ private boolean isUniqueKey(RexNode rexNode) { return false; } + private boolean isLiteralOrDynamicParams(RexNode node) { + return node instanceof RexLiteral || node instanceof RexDynamicParam; + } + private List getValuesAsList(Comparable value) { ImmutableList.Builder labelBuilder = ImmutableList.builder(); if (value instanceof NlsString) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 4a2ef40d8821..3cba4882d53a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -61,6 +61,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; /** * A unified structure to build {@link PlannerInstance} which can further build logical and physical plan from an antlr tree @@ -70,18 +71,20 @@ public class GraphPlanner { private final Configs graphConfig; private final PlannerConfig plannerConfig; private final RelOptPlanner optPlanner; - private final RexBuilder rexBuilder; private final AtomicLong idGenerator; + private final RexBuilder rexBuilder; public static final RelBuilderFactory relBuilderFactory = (RelOptCluster cluster, @Nullable RelOptSchema schema) -> GraphBuilder.create(null, (GraphOptCluster) cluster, schema); + public static final Function rexBuilderFactory = + (Configs configs) -> new GraphRexBuilder(new GraphTypeFactoryImpl(configs)); public GraphPlanner(Configs graphConfig) { this.graphConfig = graphConfig; this.plannerConfig = PlannerConfig.create(this.graphConfig); logger.debug("planner config: " + this.plannerConfig); this.optPlanner = createRelOptPlanner(this.plannerConfig); - this.rexBuilder = new GraphRexBuilder(new GraphTypeFactoryImpl(graphConfig)); + this.rexBuilder = rexBuilderFactory.apply(graphConfig); this.idGenerator = new AtomicLong(FrontendConfig.FRONTEND_SERVER_ID.get(graphConfig)); } From ae2ba09940456383915ff3a07cd8c50a48e32056 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 12 Oct 2023 12:29:32 +0800 Subject: [PATCH 10/10] [GIE Compiler] convert continuous ranges search to compositions of 'and' or 'or' --- .../ir/runtime/ffi/RelToFfiConverter.java | 30 ++-- .../ir/runtime/proto/RexToProtoConverter.java | 25 +++- .../common/ir/runtime/proto/Utils.java | 59 +++++++- .../common/ir/tools/GraphBuilder.java | 141 +++++++++++------- .../graphscope/common/ir/tools/Utils.java | 22 +++ .../common/ir/runtime/RexToProtoTest.java | 2 +- 6 files changed, 210 insertions(+), 69 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java index fb044d9e4402..02332052ab5f 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java @@ -206,7 +206,9 @@ public RelNode visit(GraphLogicalMultiMatch match) { @Override public RelNode visit(LogicalFilter logicalFilter) { OuterExpression.Expression exprProto = - logicalFilter.getCondition().accept(new RexToProtoConverter(true, isColumnId)); + logicalFilter + .getCondition() + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); Pointer ptrFilter = LIB.initSelectOperator(); checkFfiResult( LIB.setSelectPredicatePb( @@ -220,7 +222,9 @@ public PhysicalNode visit(GraphLogicalProject project) { List fields = project.getRowType().getFieldList(); for (int i = 0; i < project.getProjects().size(); ++i) { OuterExpression.Expression expression = - project.getProjects().get(i).accept(new RexToProtoConverter(true, isColumnId)); + project.getProjects() + .get(i) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); int aliasId = fields.get(i).getIndex(); FfiAlias.ByValue ffiAlias = (aliasId == AliasInference.DEFAULT_ID) @@ -261,7 +265,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { RexGraphVariable.class, var.getClass()); OuterExpression.Expression expr = - var.accept(new RexToProtoConverter(true, isColumnId)); + var.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); int aliasId; if (i >= fields.size() || (aliasId = fields.get(i).getIndex()) == AliasInference.DEFAULT_ID) { @@ -284,7 +288,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { field.getName(), field.getType()); OuterExpression.Variable exprVar = - rexVar.accept(new RexToProtoConverter(true, isColumnId)) + rexVar.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -306,7 +310,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { OuterExpression.Variable var = groupKeys .get(i) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); int aliasId = fields.get(i).getIndex(); @@ -340,7 +344,7 @@ public PhysicalNode visit(GraphLogicalAggregate aggregate) { operands.get(0).getClass()); OuterExpression.Variable var = operands.get(0) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -370,7 +374,7 @@ public PhysicalNode visit(GraphLogicalSort sort) { for (int i = 0; i < collations.size(); ++i) { RexGraphVariable expr = ((GraphFieldCollation) collations.get(i)).getVariable(); OuterExpression.Variable var = - expr.accept(new RexToProtoConverter(true, isColumnId)) + expr.accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -406,13 +410,13 @@ public PhysicalNode visit(LogicalJoin join) { OuterExpression.Variable leftVar = leftRightVars .get(0) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); OuterExpression.Variable rightVar = leftRightVars .get(1) - .accept(new RexToProtoConverter(true, isColumnId)) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)) .getOperators(0) .getVar(); checkFfiResult( @@ -452,7 +456,10 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) { }); if (ObjectUtils.isNotEmpty(tableScan.getFilters())) { OuterExpression.Expression expression = - tableScan.getFilters().get(0).accept(new RexToProtoConverter(true, isColumnId)); + tableScan + .getFilters() + .get(0) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); checkFfiResult( LIB.setParamsPredicatePb( params, new FfiPbPointer.ByValue(expression.toByteArray()))); @@ -632,7 +639,8 @@ private void addFilterToFfiBinder(Pointer ptrSentence, AbstractBindableTableScan List filters = tableScan.getFilters(); if (ObjectUtils.isNotEmpty(filters)) { OuterExpression.Expression exprProto = - filters.get(0).accept(new RexToProtoConverter(true, isColumnId)); + filters.get(0) + .accept(new RexToProtoConverter(true, isColumnId, this.rexBuilder)); Pointer ptrFilter = LIB.initSelectOperator(); checkFfiResult( LIB.setSelectPredicatePb( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java index 3e57e4e77d3f..220074c03e5d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java @@ -27,6 +27,7 @@ import org.apache.calcite.rex.*; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.util.Sarg; import java.util.List; @@ -35,10 +36,12 @@ */ public class RexToProtoConverter extends RexVisitorImpl { private final boolean isColumnId; + private final RexBuilder rexBuilder; - public RexToProtoConverter(boolean deep, boolean isColumnId) { + public RexToProtoConverter(boolean deep, boolean isColumnId, RexBuilder rexBuilder) { super(deep); this.isColumnId = isColumnId; + this.rexBuilder = rexBuilder; } @Override @@ -166,6 +169,26 @@ private OuterExpression.Expression visitIsNullOperator(RexNode operand) { } private OuterExpression.Expression visitBinaryOperator(RexCall call) { + if (call.getOperator().getKind() == SqlKind.SEARCH) { + // ir core can not support continuous ranges in a search operator, here expand it to + // compositions of 'and' or 'or', + // i.e. a.age SEARCH [[1, 10]] -> a.age >= 1 and a.age <= 10 + RexNode left = call.getOperands().get(0); + RexNode right = call.getOperands().get(1); + RexLiteral literal = null; + if (left instanceof RexLiteral) { + literal = (RexLiteral) left; + } else if (right instanceof RexLiteral) { + literal = (RexLiteral) right; + } + if (literal != null && literal.getValue() instanceof Sarg) { + Sarg sarg = (Sarg) literal.getValue(); + // search continuous ranges + if (!sarg.isPoints()) { + call = (RexCall) RexUtil.expandSearch(this.rexBuilder, null, call); + } + } + } SqlOperator operator = call.getOperator(); OuterExpression.Expression.Builder exprBuilder = OuterExpression.Expression.newBuilder(); // left-associative diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index 1245a82bb6d6..7345336353a0 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -17,7 +17,10 @@ package com.alibaba.graphscope.common.ir.runtime.proto; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; -import com.alibaba.graphscope.common.ir.type.*; +import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.alibaba.graphscope.common.ir.type.GraphNameOrId; +import com.alibaba.graphscope.common.ir.type.GraphProperty; +import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.gaia.proto.Common; import com.alibaba.graphscope.gaia.proto.DataType; import com.alibaba.graphscope.gaia.proto.GraphAlgebra; @@ -31,6 +34,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.Sarg; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,8 +73,54 @@ public static final Common.Value protoValue(RexLiteral literal) { return Common.Value.newBuilder() .setF64(((Number) literal.getValue()).doubleValue()) .build(); + case SARG: + Sarg sarg = literal.getValueAs(Sarg.class); + if (sarg.isPoints()) { + Common.Value.Builder valueBuilder = Common.Value.newBuilder(); + List values = + com.alibaba.graphscope.common.ir.tools.Utils.getValuesAsList(sarg); + if (values.isEmpty()) { + // return an empty string array to handle the case i.e. within [] + return valueBuilder + .setStrArray(Common.StringArray.newBuilder().build()) + .build(); + } + Comparable first = values.get(0); + if (first instanceof Integer) { + Common.I32Array.Builder i32Array = Common.I32Array.newBuilder(); + values.forEach(value -> i32Array.addItem((Integer) value)); + valueBuilder.setI32Array(i32Array); + } else if (first instanceof Number) { + Common.I64Array.Builder i64Array = Common.I64Array.newBuilder(); + values.forEach(value -> i64Array.addItem(((Number) value).longValue())); + valueBuilder.setI64Array(i64Array); + } else if (first instanceof Double || first instanceof Float) { + Common.DoubleArray.Builder doubleArray = Common.DoubleArray.newBuilder(); + values.forEach( + value -> + doubleArray.addItem( + (value instanceof Float) + ? (Float) value + : (Double) value)); + valueBuilder.setF64Array(doubleArray); + } else if (first instanceof String || first instanceof NlsString) { + Common.StringArray.Builder stringArray = Common.StringArray.newBuilder(); + values.forEach( + value -> + stringArray.addItem( + (value instanceof NlsString) + ? ((NlsString) value).getValue() + : (String) value)); + valueBuilder.setStrArray(stringArray); + } else { + throw new UnsupportedOperationException( + "can not convert value list=" + values + " to ir core array"); + } + return valueBuilder.build(); + } + throw new UnsupportedOperationException( + "can not convert continuous ranges to ir core structure, sarg=" + sarg); default: - // TODO: support int/double/string array throw new UnsupportedOperationException( "literal type " + literal.getTypeName() + " is unsupported yet"); } @@ -180,8 +230,11 @@ public static final OuterExpression.ExprOpr protoOperator(SqlOperator operator) return OuterExpression.ExprOpr.newBuilder() .setLogical(OuterExpression.Logical.ISNULL) .build(); + case SEARCH: + return OuterExpression.ExprOpr.newBuilder() + .setLogical(OuterExpression.Logical.WITHIN) + .build(); default: - // TODO: support IN and NOT_IN throw new UnsupportedOperationException( "operator type=" + operator.getKind() diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index d917e58d0b59..d52d233838f6 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -38,7 +38,10 @@ import com.alibaba.graphscope.common.ir.type.*; import com.alibaba.graphscope.gremlin.Utils; import com.google.common.base.Preconditions; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.calcite.plan.*; import org.apache.calcite.rel.AbstractRelNode; @@ -55,7 +58,6 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Litmus; -import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Sarg; import org.apache.commons.lang3.ObjectUtils; @@ -819,25 +821,13 @@ private void classifyFilters( List conjunctions = RelOptUtil.conjunctions(condition); List filtersToRemove = Lists.newArrayList(); for (RexNode conjunction : conjunctions) { - if (conjunction instanceof RexCall) { - RexCall rexCall = (RexCall) conjunction; - if (rexCall.getOperator().getKind() == SqlKind.EQUALS - || rexCall.getOperator().getKind() == SqlKind.SEARCH) { - RexNode left = rexCall.getOperands().get(0); - RexNode right = rexCall.getOperands().get(1); - if (left.getType() instanceof GraphLabelType && right instanceof RexLiteral) { - filtersToRemove.add(conjunction); - labelValues.addAll( - getValuesAsList(((RexLiteral) right).getValueAs(Comparable.class))); - break; - } else if (left instanceof RexLiteral - && right.getType() instanceof GraphLabelType) { - filtersToRemove.add(conjunction); - labelValues.addAll( - getValuesAsList(((RexLiteral) left).getValueAs(Comparable.class))); - break; - } - } + RexLiteral labelLiteral = isLabelEqualFilter(conjunction); + if (labelLiteral != null) { + filtersToRemove.add(conjunction); + labelValues.addAll( + com.alibaba.graphscope.common.ir.tools.Utils.getValuesAsList( + labelLiteral.getValueAs(Comparable.class))); + break; } } if (tableScan instanceof GraphLogicalSource @@ -845,20 +835,9 @@ private void classifyFilters( // try to extract unique key filters from the original condition List disjunctions = RelOptUtil.disjunctions(condition); for (RexNode disjunction : disjunctions) { - if (disjunction instanceof RexCall) { - RexCall rexCall = (RexCall) disjunction; - if (rexCall.getOperator().getKind() == SqlKind.EQUALS - || rexCall.getOperator().getKind() == SqlKind.SEARCH) { - RexNode left = rexCall.getOperands().get(0); - RexNode right = rexCall.getOperands().get(1); - if (isUniqueKey(left) && isLiteralOrDynamicParams(right)) { - filtersToRemove.add(disjunction); - uniqueKeyFilters.add(disjunction); - } else if (isLiteralOrDynamicParams(left) && isUniqueKey(right)) { - filtersToRemove.add(disjunction); - uniqueKeyFilters.add(disjunction); - } - } + if (isUniqueKeyEqualFilter(disjunction)) { + filtersToRemove.add(disjunction); + uniqueKeyFilters.add(disjunction); } } } @@ -868,6 +847,80 @@ private void classifyFilters( filters.addAll(conjunctions); } + // check the condition if it is the pattern of label equal filter, i.e. ~label = 'person' or + // ~label within ['person', 'software'] + // if it is then return the literal containing label values, otherwise null + private @Nullable RexLiteral isLabelEqualFilter(RexNode condition) { + if (condition instanceof RexCall) { + RexCall rexCall = (RexCall) condition; + SqlOperator operator = rexCall.getOperator(); + switch (operator.getKind()) { + case EQUALS: + case SEARCH: + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (left.getType() instanceof GraphLabelType && right instanceof RexLiteral) { + Comparable value = ((RexLiteral) right).getValue(); + // if Sarg is a continuous range then the filter is not the 'equal', i.e. + // ~label SEARCH [[1, 10]] which means ~label >= 1 and ~label <= 10 + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return null; + } + return (RexLiteral) right; + } else if (right.getType() instanceof GraphLabelType + && left instanceof RexLiteral) { + Comparable value = ((RexLiteral) left).getValue(); + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return null; + } + return (RexLiteral) left; + } + default: + return null; + } + } else { + return null; + } + } + + // check the condition if it is the pattern of unique key equal filter, i.e. ~id = 1 or ~id + // within [1, 2] + private boolean isUniqueKeyEqualFilter(RexNode condition) { + if (condition instanceof RexCall) { + RexCall rexCall = (RexCall) condition; + SqlOperator operator = rexCall.getOperator(); + switch (operator.getKind()) { + case EQUALS: + case SEARCH: + RexNode left = rexCall.getOperands().get(0); + RexNode right = rexCall.getOperands().get(1); + if (isUniqueKey(left) && isLiteralOrDynamicParams(right)) { + if (right instanceof RexLiteral) { + Comparable value = ((RexLiteral) right).getValue(); + // if Sarg is a continuous range then the filter is not the 'equal', + // i.e. ~id SEARCH [[1, 10]] which means ~id >= 1 and ~id <= 10 + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return false; + } + } + return true; + } else if (isUniqueKey(right) && isLiteralOrDynamicParams(left)) { + if (left instanceof RexLiteral) { + Comparable value = ((RexLiteral) left).getValue(); + if (value instanceof Sarg && !((Sarg) value).isPoints()) { + return false; + } + } + return true; + } + default: + return false; + } + } else { + return false; + } + } + private boolean isUniqueKey(RexNode rexNode) { // todo: support primary keys if (rexNode instanceof RexGraphVariable) { @@ -882,24 +935,6 @@ private boolean isLiteralOrDynamicParams(RexNode node) { return node instanceof RexLiteral || node instanceof RexDynamicParam; } - private List getValuesAsList(Comparable value) { - ImmutableList.Builder labelBuilder = ImmutableList.builder(); - if (value instanceof NlsString) { - labelBuilder.add(((NlsString) value).getValue()); - } else if (value instanceof Sarg) { - Sarg sarg = (Sarg) value; - if (sarg.isPoints()) { - Set> rangeSets = sarg.rangeSet.asRanges(); - for (Range range : rangeSets) { - labelBuilder.addAll(getValuesAsList(range.lowerEndpoint())); - } - } - } else { - labelBuilder.add(value); - } - return labelBuilder.build(); - } - // return the top node if its type is Filter, otherwise null private Filter topFilter() { if (this.size() > 0 && this.peek() instanceof Filter) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java index 701cc3ebb337..ea970934c9fd 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java @@ -16,7 +16,9 @@ package com.alibaba.graphscope.common.ir.tools; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.apache.calcite.rel.RelNode; @@ -24,6 +26,8 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelRecordType; import org.apache.calcite.rel.type.StructKind; +import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.Sarg; import java.util.List; import java.util.Set; @@ -53,4 +57,22 @@ public static RelDataType getOutputType(RelNode topNode) { .collect(Collectors.toList()); return new RelRecordType(StructKind.FULLY_QUALIFIED, dedup); } + + public static List getValuesAsList(Comparable value) { + ImmutableList.Builder valueBuilder = ImmutableList.builder(); + if (value instanceof NlsString) { + valueBuilder.add(((NlsString) value).getValue()); + } else if (value instanceof Sarg) { + Sarg sarg = (Sarg) value; + if (sarg.isPoints()) { + Set> rangeSets = sarg.rangeSet.asRanges(); + for (Range range : rangeSets) { + valueBuilder.addAll(getValuesAsList(range.lowerEndpoint())); + } + } + } else { + valueBuilder.add(value); + } + return valueBuilder.build(); + } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java index 311f304f318a..191e112a274c 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/RexToProtoTest.java @@ -50,7 +50,7 @@ public void test_expression_with_brace() throws Exception { GraphStdOperatorTable.MINUS, builder.literal(1), builder.variable("a", "age"))); - RexToProtoConverter converter = new RexToProtoConverter(true, false); + RexToProtoConverter converter = new RexToProtoConverter(true, false, Utils.rexBuilder); Assert.assertEquals( FileUtils.readJsonFromResource("proto/expression_with_brace.json"), JsonFormat.printer().print(braceExpr.accept(converter)));