diff --git a/docs/interactive_engine/neo4j/supported_cypher.md b/docs/interactive_engine/neo4j/supported_cypher.md index faf0c0319c9f..0f627d65d458 100644 --- a/docs/interactive_engine/neo4j/supported_cypher.md +++ b/docs/interactive_engine/neo4j/supported_cypher.md @@ -110,6 +110,11 @@ Note that some Aggregator operators, such as `max()`, we listed here are impleme | elementId | Get a vertex or an edge identifier, unique by an object type and a database | elementId() | elementId() | | | | Type | Get label name of an edge type | type() | type() | | | | Extract | Get interval value from a temporal type | \.\ | \.\ | | | +| User Defined Functions | get all edges from a path | relationships(path) | gs.function.relationships(path) | | | +| User Defined Functions | get all nodes from a path | nodes(path) | gs.function.nodes(path) | | | +| User Defined Functions | get start node from an edge | startNode(edge) | gs.function.startNode(edge) | | | +| User Defined Functions | get end node from an edge | endNode(edge) | gs.function.endNode(edge) | | | +| User Defined Functions | convert integer value to datetime | datetime(1287230400000) | gs.function.datetime(1287230400000) | | | ## Clause A notable limitation for now is that we do not diff --git a/interactive_engine/compiler/src/main/antlr4/ExprGS.g4 b/interactive_engine/compiler/src/main/antlr4/ExprGS.g4 index 11546bd61cb3..f83dde542297 100644 --- a/interactive_engine/compiler/src/main/antlr4/ExprGS.g4 +++ b/interactive_engine/compiler/src/main/antlr4/ExprGS.g4 @@ -106,7 +106,7 @@ oC_Atom // todo: support user defined function oC_FunctionInvocation - : oC_AggregateFunctionInvocation | oC_ScalarFunctionInvocation; + : oC_AggregateFunctionInvocation | oC_ScalarFunctionInvocation | oC_UserDefinedFunctionInvocation; oC_AggregateFunctionInvocation : ( COUNT | SUM | MIN | MAX | COLLECT | AVG | FOLD | MEAN ) SP? '(' SP? ( DISTINCT SP? )? ( oC_Expression SP? ( ',' SP? oC_Expression SP? )* )? ')' ; @@ -144,8 +144,12 @@ HEAD : ( 'H' | 'h' ) ( 'E' | 'e' ) ( 'A' | 'a' ) ( 'D' | 'd' ); DURATION: ( 'D' | 'd' ) ( 'U' | 'u' ) ( 'R' | 'r' ) ( 'A' | 'a' ) ( 'T' | 't' ) ( 'I' | 'i' ) ( 'O' | 'o' ) ( 'N' | 'n' ); -oC_FunctionName - : oC_Namespace oC_SymbolicName ; +// user defined function should start with a namespace 'gs.function.' +oC_UserDefinedFunctionInvocation + : oC_UserDefinedFunctionName SP? '(' SP? ( oC_Expression SP? ( ',' SP? oC_Expression SP? )* )? ')' ; + +oC_UserDefinedFunctionName + : 'gs.function.' oC_Namespace oC_SymbolicName ; oC_Namespace : ( oC_SymbolicName '.' )* ; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java index 20e1e35a6bee..9ab065f61366 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java @@ -34,4 +34,7 @@ public class GraphConfig { // an intermediate solution to support foreign key, will be integrated into schema public static final Config GRAPH_FOREIGN_KEY_URI = Config.stringConfig("graph.foreign.key", ""); + + public static final Config GRAPH_FUNCTIONS_URI = + Config.stringConfig("graph.functions", ""); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/BuiltInFunction.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/BuiltInFunction.java new file mode 100644 index 000000000000..f60417db08a0 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/BuiltInFunction.java @@ -0,0 +1,138 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.function; + +import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.alibaba.graphscope.common.ir.type.GraphPathType; +import com.alibaba.graphscope.common.ir.type.GraphSchemaType; +import com.alibaba.graphscope.common.ir.type.GraphTypeFamily; +import com.google.common.collect.ImmutableList; + +import org.apache.calcite.sql.type.*; + +import java.util.List; +import java.util.stream.Collectors; + +public enum BuiltInFunction { + GS_FUNCTION_RELATIONSHIPS( + "gs.function.relationships", + // set return type inference of function 'gs.function.relationships': + // 1. get first parameter type which should be `GraphPathType` + // 2. get the inner edge type of `GraphPathType` + // 3. convert to array type which has the edge type as the component + ReturnTypes.ARG0 + .andThen( + (op, type) -> { + GraphPathType.ElementType elementType = + (GraphPathType.ElementType) type.getComponentType(); + return elementType.getExpandType(); + }) + .andThen(SqlTypeTransforms.TO_ARRAY), + // set operand type checker of function 'gs.function.relationships' + GraphOperandTypes.family(GraphTypeFamily.PATH), + GraphInferTypes.FIRST_KNOWN), + + GS_FUNCTION_NODES( + "gs.function.nodes", + ReturnTypes.ARG0 + .andThen( + (op, type) -> { + GraphPathType.ElementType elementType = + (GraphPathType.ElementType) type.getComponentType(); + return elementType.getGetVType(); + }) + .andThen(SqlTypeTransforms.TO_ARRAY), + GraphOperandTypes.family(GraphTypeFamily.PATH), + GraphInferTypes.FIRST_KNOWN), + + GS_FUNCTION_START_NODE( + "gs.function.startNode", + ReturnTypes.ARG0.andThen( + (op, type) -> { + GraphSchemaType edgeType = (GraphSchemaType) type; + List srcLabels = + edgeType.getLabelType().getLabelsEntry().stream() + .map( + e -> + new GraphLabelType.Entry() + .label(e.getSrcLabel()) + .labelId(e.getSrcLabelId())) + .collect(Collectors.toList()); + return new GraphSchemaType( + GraphOpt.Source.VERTEX, + new GraphLabelType(srcLabels), + ImmutableList.of()); + }), + GraphOperandTypes.family(GraphTypeFamily.EDGE), + GraphInferTypes.FIRST_KNOWN), + + GS_FUNCTION_END_NODE( + "gs.function.endNode", + ReturnTypes.ARG0.andThen( + (op, type) -> { + GraphSchemaType edgeType = (GraphSchemaType) type; + List endLabels = + edgeType.getLabelType().getLabelsEntry().stream() + .map( + e -> + new GraphLabelType.Entry() + .label(e.getDstLabel()) + .labelId(e.getDstLabelId())) + .collect(Collectors.toList()); + return new GraphSchemaType( + GraphOpt.Source.VERTEX, + new GraphLabelType(endLabels), + ImmutableList.of()); + }), + GraphOperandTypes.family(GraphTypeFamily.EDGE), + GraphInferTypes.FIRST_KNOWN); + + BuiltInFunction( + String signature, + SqlReturnTypeInference returnTypeInference, + SqlOperandTypeChecker operandTypeChecker, + SqlOperandTypeInference operandTypeInference) { + this.signature = signature; + this.returnTypeInference = returnTypeInference; + this.operandTypeChecker = operandTypeChecker; + this.operandTypeInference = operandTypeInference; + } + + private final String signature; + private final SqlReturnTypeInference returnTypeInference; + private final SqlOperandTypeChecker operandTypeChecker; + private final SqlOperandTypeInference operandTypeInference; + + public String getSignature() { + return this.signature; + } + + public SqlReturnTypeInference getReturnTypeInference() { + return this.returnTypeInference; + } + + public SqlOperandTypeChecker getOperandTypeChecker() { + return this.operandTypeChecker; + } + + public SqlOperandTypeInference getOperandTypeInference() { + return this.operandTypeInference; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/FunctionMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/FunctionMeta.java new file mode 100644 index 000000000000..e731458153b8 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/FunctionMeta.java @@ -0,0 +1,78 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.function; + +import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; + +import org.apache.calcite.sql.type.*; + +import java.util.stream.Collectors; + +public class FunctionMeta { + private final String signature; + private final SqlReturnTypeInference returnTypeInference; + private final SqlOperandTypeChecker operandTypeChecker; + private final SqlOperandTypeInference operandTypeInference; + + public FunctionMeta( + String signature, + SqlReturnTypeInference returnTypeInference, + SqlOperandTypeChecker operandTypeChecker, + SqlOperandTypeInference operandTypeInference) { + this.signature = signature; + this.returnTypeInference = returnTypeInference; + this.operandTypeChecker = operandTypeChecker; + this.operandTypeInference = operandTypeInference; + } + + public FunctionMeta(BuiltInFunction function) { + this( + function.getSignature(), + function.getReturnTypeInference(), + function.getOperandTypeChecker(), + function.getOperandTypeInference()); + } + + public FunctionMeta(StoredProcedureMeta meta) { + this( + meta.getName(), + ReturnTypes.explicit(meta.getReturnType().getFieldList().get(0).getType()), + GraphOperandTypes.metaTypeChecker(meta), + InferTypes.explicit( + meta.getParameters().stream() + .map(k -> k.getDataType()) + .collect(Collectors.toList()))); + } + + public String getSignature() { + return this.signature; + } + + public SqlReturnTypeInference getReturnTypeInference() { + return this.returnTypeInference; + } + + public SqlOperandTypeChecker getOperandTypeChecker() { + return this.operandTypeChecker; + } + + public SqlOperandTypeInference getOperandTypeInference() { + return this.operandTypeInference; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java new file mode 100644 index 000000000000..21374d84b07d --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java @@ -0,0 +1,119 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.function; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; +import com.google.common.collect.Maps; + +import org.apache.calcite.sql.type.GraphInferTypes; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.yaml.snakeyaml.Yaml; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +public class GraphFunctions { + public static final String FUNCTION_PREFIX = "gs.function."; + private final Map functionMetaMap; + + private static GraphFunctions instance; + + public static GraphFunctions instance(Configs configs) { + if (instance == null) { + instance = new GraphFunctions(configs); + } + return instance; + } + + private GraphFunctions(Configs configs) { + this.functionMetaMap = Maps.newHashMap(); + this.registerBuiltInFunctions(); + this.registerConfigFunctions(configs); + } + + // initialize built-in functions + private void registerBuiltInFunctions() { + for (BuiltInFunction function : BuiltInFunction.values()) { + FunctionMeta meta = new FunctionMeta(function); + functionMetaMap.put(function.getSignature(), meta); + } + } + + // initialize functions from configuration: + // 1. load the functions from the settings of 'graph.functions' + // 2. if the setting not found, load the default resource under the classpath + private void registerConfigFunctions(Configs configs) { + try (InputStream stream = loadConfigAsStream(configs)) { + Yaml yaml = new Yaml(); + Map map = yaml.load(stream); + if (map != null) { + Object functions = map.get("graph_functions"); + if (functions instanceof List) { + for (Object function : (List) functions) { + String functionYaml = yaml.dump(function); + StoredProcedureMeta meta = + StoredProcedureMeta.Deserializer.perform( + new ByteArrayInputStream( + functionYaml.getBytes(StandardCharsets.UTF_8))); + functionMetaMap.put(meta.getName(), new FunctionMeta(meta)); + } + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private InputStream loadConfigAsStream(Configs configs) throws Exception { + String functionConfig = GraphConfig.GRAPH_FUNCTIONS_URI.get(configs); + if (!functionConfig.isEmpty()) { + File file = new File(functionConfig); + if (file.exists()) { + return new FileInputStream(file); + } + } + return Thread.currentThread() + .getContextClassLoader() + .getResourceAsStream("conf/graph_functions.yaml"); + } + + public FunctionMeta getFunction(String functionName) { + FunctionMeta meta = functionMetaMap.get(functionName); + if (meta == null) { + // if not exist, create a new function meta with no constraints on operands and return + // types + meta = + new FunctionMeta( + functionName, + ReturnTypes.explicit(SqlTypeName.ANY), + OperandTypes.VARIADIC, + GraphInferTypes.FIRST_KNOWN); + } + return meta; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java index 92f505312157..b239fb97581d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.common.ir.runtime.proto; +import com.alibaba.graphscope.common.ir.meta.function.GraphFunctions; import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; import com.alibaba.graphscope.common.ir.tools.AliasInference; import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; @@ -33,6 +34,7 @@ import org.apache.calcite.util.Sarg; import java.util.List; +import java.util.stream.Collectors; /** * convert an expression in calcite to logical expression in ir_core @@ -70,6 +72,9 @@ public OuterExpression.Expression visitCall(RexCall call) { } else if (operator.getKind() == SqlKind.OTHER && operator.getName().equals("DATETIME_MINUS")) { return visitDateMinus(call); + } else if (operator.getKind() == SqlKind.OTHER + && operator.getName().startsWith(GraphFunctions.FUNCTION_PREFIX)) { + return visitUserDefinedFunction(call); } else if (call.getOperands().size() == 1) { return visitUnaryOperator(call); } else { @@ -77,6 +82,22 @@ public OuterExpression.Expression visitCall(RexCall call) { } } + private OuterExpression.Expression visitUserDefinedFunction(RexCall call) { + List operands = call.getOperands(); + List parameters = + operands.stream().map(operand -> operand.accept(this)).collect(Collectors.toList()); + return OuterExpression.Expression.newBuilder() + .addOperators( + OuterExpression.ExprOpr.newBuilder() + .setUdfFunc( + OuterExpression.UserDefinedFunction.newBuilder() + .setName(call.op.getName()) + .addAllParameters(parameters) + .build()) + .setNodeType(Utils.protoIrDataType(call.getType(), isColumnId))) + .build(); + } + private OuterExpression.Expression visitPathFunction(RexCall call) { List operands = call.getOperands(); RexGraphVariable variable = (RexGraphVariable) operands.get(0); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java index 41900d4a1cb4..6389c59eddb4 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java @@ -21,6 +21,7 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.exception.FrontendException; +import com.alibaba.graphscope.common.ir.meta.function.GraphFunctions; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; import com.alibaba.graphscope.common.ir.meta.schema.GraphOptSchema; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; @@ -872,10 +873,11 @@ private boolean isCurrentSupported(SqlOperator operator) { || sqlKind == SqlKind.BIT_OR || sqlKind == SqlKind.BIT_XOR || (sqlKind == SqlKind.OTHER - && (operator.getName().equals("IN") - || operator.getName().equals("DATETIME_MINUS") - || operator.getName().equals("PATH_CONCAT") - || operator.getName().equals("PATH_FUNCTION"))) + && (operator.getName().equals("IN") + || operator.getName().equals("DATETIME_MINUS") + || operator.getName().equals("PATH_CONCAT") + || operator.getName().equals("PATH_FUNCTION")) + || operator.getName().startsWith(GraphFunctions.FUNCTION_PREFIX)) || sqlKind == SqlKind.ARRAY_CONCAT; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java index ec7277105c96..16a16f515f00 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.common.ir.tools; +import com.alibaba.graphscope.common.ir.meta.function.FunctionMeta; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; import com.alibaba.graphscope.common.ir.rex.operator.CaseOperator; import com.alibaba.graphscope.common.ir.rex.operator.SqlArrayValueConstructor; @@ -29,9 +30,6 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.*; -import java.util.List; -import java.util.stream.Collectors; - /** * Extends {@link org.apache.calcite.sql.fun.SqlStdOperatorTable} to re-implement type checker/inference in some operators */ @@ -249,18 +247,7 @@ public class GraphStdOperatorTable extends SqlStdOperatorTable { public static final SqlFunction USER_DEFINED_PROCEDURE(StoredProcedureMeta meta) { SqlReturnTypeInference returnTypeInference = ReturnTypes.explicit(meta.getReturnType()); - List parameters = meta.getParameters(); - SqlOperandTypeChecker operandTypeChecker = - GraphOperandTypes.operandMetadata( - parameters.stream() - .map(p -> p.getDataType().getSqlTypeName().getFamily()) - .collect(Collectors.toList()), - typeFactory -> - parameters.stream() - .map(p -> p.getDataType()) - .collect(Collectors.toList()), - i -> parameters.get(i).getName(), - i -> false); + SqlOperandTypeChecker operandTypeChecker = GraphOperandTypes.metaTypeChecker(meta); return new SqlFunction( meta.getName(), SqlKind.PROCEDURE_CALL, @@ -270,6 +257,16 @@ public static final SqlFunction USER_DEFINED_PROCEDURE(StoredProcedureMeta meta) SqlFunctionCategory.USER_DEFINED_PROCEDURE); } + public static final SqlFunction USER_DEFINED_FUNCTION(FunctionMeta meta) { + return new SqlFunction( + meta.getSignature(), + SqlKind.OTHER, + meta.getReturnTypeInference(), + meta.getOperandTypeInference(), + meta.getOperandTypeChecker(), + SqlFunctionCategory.USER_DEFINED_FUNCTION); + } + // combine multiple expressions into a list public static final SqlOperator ARRAY_VALUE_CONSTRUCTOR = new SqlArrayValueConstructor(); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java index fe365f2691d8..209e9c317b09 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java @@ -18,6 +18,8 @@ import com.alibaba.graphscope.common.antlr4.ExprUniqueAliasInfer; import com.alibaba.graphscope.common.antlr4.ExprVisitorResult; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.function.GraphFunctions; import com.alibaba.graphscope.common.ir.rel.type.group.GraphAggCall; import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; import com.alibaba.graphscope.common.ir.rex.RexTmpVariable; @@ -233,20 +235,38 @@ public ExprVisitorResult visitOC_PropertyOrLabelsExpression( if (ctx.oC_Atom().oC_Literal() != null) { throw new IllegalArgumentException("cannot get property from an literal"); } else { - String aliasName = ctx.oC_Atom().oC_Variable().getText(); - RexGraphVariable variable = builder.variable(aliasName); String propertyName = ctx.oC_PropertyLookup().oC_PropertyKeyName().getText(); - RexNode expr = - (variable.getType() instanceof GraphSchemaType) - ? builder.variable(aliasName, propertyName) - : builder.call( - GraphStdOperatorTable.EXTRACT, - Utils.createIntervalExpr( - null, - Utils.createExtractUnit(propertyName), - builder), - variable); - return new ExprVisitorResult(expr); + ExprVisitorResult exprRes = visitOC_Atom(ctx.oC_Atom()); + RexNode expr = exprRes.getExpr(); + // get property from a vertex or an edge + if (expr.getType() instanceof GraphSchemaType) { + Preconditions.checkArgument( + expr instanceof RexGraphVariable, + "can not get property from the rex=", + expr); + String aliasName = ((RexGraphVariable) expr).getName().split("\\.")[0]; + return new ExprVisitorResult( + exprRes.getAggCalls(), builder.variable(aliasName, propertyName)); + } + // get interval from a date time + if (SqlTypeFamily.DATETIME + .getTypeNames() + .contains(expr.getType().getSqlTypeName())) { + return new ExprVisitorResult( + exprRes.getAggCalls(), + builder.call( + GraphStdOperatorTable.EXTRACT, + Utils.createIntervalExpr( + null, Utils.createExtractUnit(propertyName), builder), + expr)); + } + throw new IllegalArgumentException( + "invalid property lookup operation, cannot get property or extract interval" + + " from expr=[" + + expr + + ", type=" + + expr.getType() + + "]"); } } } @@ -403,6 +423,29 @@ public ExprVisitorResult visitOC_AggregateFunctionInvocation( RexTmpVariable.of(alias, ((GraphAggCall) aggCall).getType())); } + @Override + public ExprVisitorResult visitOC_UserDefinedFunctionInvocation( + CypherGSParser.OC_UserDefinedFunctionInvocationContext ctx) { + String functionName = ctx.oC_UserDefinedFunctionName().getText(); + List aggCalls = Lists.newArrayList(); + List parameters = Lists.newArrayList(); + ctx.oC_Expression() + .forEach( + k -> { + ExprVisitorResult res = visitOC_Expression(k); + aggCalls.addAll(res.getAggCalls()); + parameters.add(res.getExpr()); + }); + Configs configs = parent.getGraphBuilder().getContext().unwrapOrThrow(Configs.class); + GraphFunctions functions = GraphFunctions.instance(configs); + RexNode udfCall = + builder.call( + GraphStdOperatorTable.USER_DEFINED_FUNCTION( + functions.getFunction(functionName)), + parameters); + return new ExprVisitorResult(aggCalls, udfCall); + } + @Override public ExprVisitorResult visitOC_ScalarFunctionInvocation( CypherGSParser.OC_ScalarFunctionInvocationContext ctx) { diff --git a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java index 76dfa03a2b87..5a9a93dc663f 100644 --- a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java +++ b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java @@ -16,7 +16,6 @@ package org.apache.calcite.sql.type; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.calcite.linq4j.function.Functions; @@ -42,11 +41,11 @@ public class GraphOperandMetaDataImpl extends GraphFamilyOperandTypeChecker private final IntFunction paramNameFn; GraphOperandMetaDataImpl( - List expectedexpectedFamilies, + List expectedFamilies, Function<@Nullable RelDataTypeFactory, List> paramTypesFactory, IntFunction paramNameFn, Predicate optional) { - super(expectedexpectedFamilies, optional); + super(expectedFamilies, optional); this.paramTypesFactory = Objects.requireNonNull(paramTypesFactory, "paramTypesFactory"); this.paramNameFn = paramNameFn; } @@ -55,13 +54,18 @@ public class GraphOperandMetaDataImpl extends GraphFamilyOperandTypeChecker protected Collection getAllowedTypeNames( RelDataTypeFactory typeFactory, SqlTypeFamily family, int iFormalOperand) { List paramsAllowedTypes = paramTypes(typeFactory); - Preconditions.checkArgument( - paramsAllowedTypes.size() > iFormalOperand, + if (paramsAllowedTypes.size() > iFormalOperand) { + return ImmutableList.of(paramsAllowedTypes.get(iFormalOperand).getSqlTypeName()); + } else if (expectedFamilies.get(iFormalOperand) instanceof SqlTypeFamily) { + return ((SqlTypeFamily) expectedFamilies.get(iFormalOperand)).getTypeNames(); + } + throw new IllegalArgumentException( "cannot find allowed type for type index=" + iFormalOperand + " from the allowed types list=" - + paramsAllowedTypes); - return ImmutableList.of(paramsAllowedTypes.get(iFormalOperand).getSqlTypeName()); + + paramsAllowedTypes + + " or expected families=" + + expectedFamilies); } @Override diff --git a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java index 8d06e8b41902..b75b862c8530 100644 --- a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java +++ b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java @@ -16,6 +16,7 @@ package org.apache.calcite.sql.type; +import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; import com.google.common.collect.ImmutableList; import org.apache.calcite.rel.type.RelDataType; @@ -26,6 +27,7 @@ import java.util.function.Function; import java.util.function.IntFunction; import java.util.function.Predicate; +import java.util.stream.Collectors; /** * Similar to {@link OperandTypes}, but we rewrite some {@link SqlOperandTypeChecker} @@ -79,7 +81,7 @@ public abstract class GraphOperandTypes { * @param families * @return */ - public static FamilyOperandTypeChecker family(SqlTypeFamily... families) { + public static FamilyOperandTypeChecker family(RelDataTypeFamily... families) { return new GraphFamilyOperandTypeChecker(ImmutableList.copyOf(families), i -> false); } @@ -103,4 +105,16 @@ public static SqlOperandMetadata operandMetadata( Predicate optional) { return new GraphOperandMetaDataImpl(families, typesFactory, operandName, optional); } + + public static SqlOperandTypeChecker metaTypeChecker(StoredProcedureMeta meta) { + List parameters = meta.getParameters(); + return GraphOperandTypes.operandMetadata( + parameters.stream() + .map(p -> p.getDataType().getSqlTypeName().getFamily()) + .collect(Collectors.toList()), + typeFactory -> + parameters.stream().map(p -> p.getDataType()).collect(Collectors.toList()), + i -> parameters.get(i).getName(), + i -> false); + } } diff --git a/interactive_engine/compiler/src/main/resources/conf/graph_functions.yaml b/interactive_engine/compiler/src/main/resources/conf/graph_functions.yaml new file mode 100644 index 000000000000..2da555e60c4c --- /dev/null +++ b/interactive_engine/compiler/src/main/resources/conf/graph_functions.yaml @@ -0,0 +1,22 @@ +graph_functions: + - name: gs.function.datetime + description: "" + params: + - name: value + type: + primitive_type: DT_SIGNED_INT32 + returns: + - name: timestamp + type: + temporal: + timestamp: + - name: gs.function.toFloat + description: "" + params: + - name: integer + type: + primitive_type: DT_SIGNED_INT32 + returns: + - name: float + type: + primitive_type: DT_FLOAT \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java index 17ba8e808dc9..417d7f4f4995 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java @@ -19,6 +19,9 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.exception.FrontendException; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; +import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; @@ -28,9 +31,34 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlTypeName; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class MatchTest { + private static Configs configs; + private static IrMeta irMeta; + private static GraphRelOptimizer optimizer; + + @BeforeClass + public static void beforeClass() { + configs = + new Configs( + ImmutableMap.of( + "graph.planner.is.on", + "true", + "graph.planner.opt", + "CBO", + "graph.planner.rules", + "FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule," + + " ExpandGetVFusionRule")); + optimizer = new GraphRelOptimizer(configs); + irMeta = + com.alibaba.graphscope.common.ir.Utils.mockIrMeta( + "schema/modern.json", + "statistics/modern_statistics.json", + optimizer.getGlogueHolder()); + } + @Test public void match_1_test() { RelNode source = Utils.eval("Match (n) Return n").build(); @@ -524,4 +552,59 @@ public void property_exist_after_type_inference_test() { + " properties=[BIGINT creationDate]) h)", rel.getRowType().toString()); } + + @Test + public void udf_function_test() { + GraphBuilder builder = + com.alibaba.graphscope.common.ir.Utils.mockGraphBuilder(optimizer, irMeta); + RelNode node = + Utils.eval( + "MATCH (person1:person)-[path:knows]->(person2:person)\n" + + " Return gs.function.startNode(path)", + builder) + .build(); + RelNode after = optimizer.optimize(node, new GraphIOProcessor(builder, irMeta)); + Assert.assertEquals( + "GraphLogicalProject($f0=[gs.function.startNode(path)], isAppend=[false])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," + + " alias=[person2], opt=[END])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[knows]}]," + + " alias=[path], startAlias=[person1], opt=[OUT])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," + + " alias=[person1], opt=[VERTEX])", + after.explain().trim()); + + RelNode node2 = + Utils.eval( + "Match (person)-[:created]->(software) WITH software.creationDate" + + " as date1\n" + + "Return 12 * (date1.year - gs.function.datetime($date2).year)" + + " + (date1.month - gs.function.datetime($date2).month)", + builder) + .build(); + RelNode after2 = optimizer.optimize(node2, new GraphIOProcessor(builder, irMeta)); + Assert.assertEquals( + "GraphLogicalProject($f0=[+(*(12, -(EXTRACT(FLAG(YEAR), date1), EXTRACT(FLAG(YEAR)," + + " gs.function.datetime(?0)))), -(EXTRACT(FLAG(MONTH), date1)," + + " EXTRACT(FLAG(MONTH), gs.function.datetime(?0))))], isAppend=[false])\n" + + " GraphLogicalProject(date1=[software.creationDate], isAppend=[false])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false, tables=[created]}]," + + " alias=[software], startAlias=[person], opt=[OUT], physicalOpt=[VERTEX])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," + + " alias=[person], opt=[VERTEX])", + after2.explain().trim()); + + RelNode node3 = + Utils.eval( + "Match (person:person)\n" + + "Return gs.function.toFloat(person.age)", + builder) + .build(); + RelNode after3 = optimizer.optimize(node3, new GraphIOProcessor(builder, irMeta)); + Assert.assertEquals( + "GraphLogicalProject($f0=[gs.function.toFloat(person.age)], isAppend=[false])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," + + " alias=[person], opt=[VERTEX])", + after3.explain().trim()); + } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java index 37cb5878df82..3c1ae5115f7d 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java @@ -21,7 +21,6 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.rules.CoreRules; -import org.apache.calcite.runtime.CalciteException; import org.junit.Assert; import org.junit.Test; @@ -261,14 +260,13 @@ public void where_10_test() { + " creationDate.month > 1990 and creationDate.day = 12" + " Return creationDate") .build(); - } catch (CalciteException e) { + } catch (Exception e) { Assert.assertTrue( e.getMessage() .contains( - "Cannot apply EXTRACT to arguments of type 'EXTRACT(, )'. Supported form(s):" - + " 'EXTRACT(, )'\n" - + "'EXTRACT(, )'")); + "invalid property lookup operation, cannot get property or" + + " extract interval from expr=[creationDate," + + " type=CHAR(1)]")); return; } Assert.fail("should have thrown exceptions for property 'name' is not a date type"); diff --git a/interactive_engine/executor/ir/proto/expr.proto b/interactive_engine/executor/ir/proto/expr.proto index 2e2d10cf5c0e..52710d425200 100644 --- a/interactive_engine/executor/ir/proto/expr.proto +++ b/interactive_engine/executor/ir/proto/expr.proto @@ -248,6 +248,11 @@ message PathConcat { ConcatPathInfo right = 2; } +message UserDefinedFunction { + string name = 1; + repeated Expression parameters = 2; +} + // An operator of expression is one of Logical, Arithmetic, Const and Variable. message ExprOpr { enum Brace { @@ -272,6 +277,7 @@ message ExprOpr { DateTimeMinus date_time_minus = 15; PathConcat path_concat = 16; PathFunction path_func = 17; + UserDefinedFunction udf_func = 18; } // The data of type of ExprOpr common.IrDataType node_type = 12;