refactor(interactive): Support User Defined Functions in Compiler

docs/interactive_engine/neo4j/supported_cypher.md

@@ -110,6 +110,11 @@ Note that some Aggregator operators, such as `max()`, we listed here are impleme
 | elementId | Get a vertex or an edge identifier, unique by an object type and a database | elementId() | elementId() | <input type="checkbox" disabled checked /> |   |
 | Type | Get label name of an edge type | type() | type() | <input type="checkbox" disabled checked /> |   |
 | Extract | Get interval value from a temporal type | \<temporal\>.\<interval\> | \<temporal\>.\<interval\> | <input type="checkbox" disabled checked /> |   |
+| User Defined Functions | get all edges from a path | relationships(path) | gs.function.relationships(path) | <input type="checkbox" disabled checked /> |   |
+| User Defined Functions | get all nodes from a path | nodes(path) | gs.function.nodes(path) | <input type="checkbox" disabled checked /> |   |
+| User Defined Functions | get start node from an edge | startNode(edge) | gs.function.startNode(edge) | <input type="checkbox" disabled checked /> |   |
+| User Defined Functions | get end node from an edge | endNode(edge) | gs.function.endNode(edge) | <input type="checkbox" disabled checked /> |   |
+| User Defined Functions | convert integer value to datetime | datetime(1287230400000) | gs.function.datetime(1287230400000) | <input type="checkbox" disabled checked /> |   |
 
 ## Clause
 A notable limitation for now is that we do not

interactive_engine/compiler/src/main/antlr4/ExprGS.g4

@@ -106,7 +106,7 @@ oC_Atom
 
 // todo: support user defined function
 oC_FunctionInvocation
-    :  oC_AggregateFunctionInvocation | oC_ScalarFunctionInvocation;
+    :  oC_AggregateFunctionInvocation | oC_ScalarFunctionInvocation | oC_UserDefinedFunctionInvocation;
 
 oC_AggregateFunctionInvocation
     :  ( COUNT | SUM | MIN | MAX | COLLECT | AVG | FOLD | MEAN ) SP? '(' SP? ( DISTINCT SP? )? ( oC_Expression SP? ( ',' SP? oC_Expression SP? )* )? ')' ;
@@ -144,8 +144,12 @@ HEAD : ( 'H' | 'h' ) ( 'E' | 'e' ) ( 'A' | 'a' ) ( 'D' | 'd' );
 
 DURATION: ( 'D' | 'd' ) ( 'U' | 'u' ) ( 'R' | 'r' ) ( 'A' | 'a' ) ( 'T' | 't' ) ( 'I' | 'i' ) ( 'O' | 'o' ) ( 'N' | 'n' );
 
-oC_FunctionName
-    :  oC_Namespace oC_SymbolicName ;
+// user defined function should start with a namespace 'gs.function.'
+oC_UserDefinedFunctionInvocation
+    : oC_UserDefinedFunctionName SP? '(' SP? ( oC_Expression SP? ( ',' SP? oC_Expression SP? )* )? ')' ;
+
+oC_UserDefinedFunctionName
+    : 'gs.function.' oC_Namespace oC_SymbolicName ;
 
 oC_Namespace
      :  ( oC_SymbolicName '.' )* ;

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java

@@ -34,4 +34,7 @@ public class GraphConfig {
     // an intermediate solution to support foreign key, will be integrated into schema
     public static final Config<String> GRAPH_FOREIGN_KEY_URI =
             Config.stringConfig("graph.foreign.key", "");
+
+    public static final Config<String> GRAPH_FUNCTIONS_URI =
+            Config.stringConfig("graph.functions", "");
 }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/BuiltInFunction.java

@@ -0,0 +1,138 @@
+/*
+ *
+ *  * Copyright 2020 Alibaba Group Holding Limited.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  * http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package com.alibaba.graphscope.common.ir.meta.function;
+
+import com.alibaba.graphscope.common.ir.tools.config.GraphOpt;
+import com.alibaba.graphscope.common.ir.type.GraphLabelType;
+import com.alibaba.graphscope.common.ir.type.GraphPathType;
+import com.alibaba.graphscope.common.ir.type.GraphSchemaType;
+import com.alibaba.graphscope.common.ir.type.GraphTypeFamily;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.calcite.sql.type.*;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public enum BuiltInFunction {
+    GS_FUNCTION_RELATIONSHIPS(
+            "gs.function.relationships",
+            // set return type inference of function 'gs.function.relationships':
+            // 1. get first parameter type which should be `GraphPathType`
+            // 2. get the inner edge type of `GraphPathType`
+            // 3. convert to array type which has the edge type as the component
+            ReturnTypes.ARG0
+                    .andThen(
+                            (op, type) -> {
+                                GraphPathType.ElementType elementType =
+                                        (GraphPathType.ElementType) type.getComponentType();
+                                return elementType.getExpandType();
+                            })
+                    .andThen(SqlTypeTransforms.TO_ARRAY),
+            // set operand type checker of function 'gs.function.relationships'
+            GraphOperandTypes.family(GraphTypeFamily.PATH),
+            GraphInferTypes.FIRST_KNOWN),
+
+    GS_FUNCTION_NODES(
+            "gs.function.nodes",
+            ReturnTypes.ARG0
+                    .andThen(
+                            (op, type) -> {
+                                GraphPathType.ElementType elementType =
+                                        (GraphPathType.ElementType) type.getComponentType();
+                                return elementType.getGetVType();
+                            })
+                    .andThen(SqlTypeTransforms.TO_ARRAY),
+            GraphOperandTypes.family(GraphTypeFamily.PATH),
+            GraphInferTypes.FIRST_KNOWN),
+
+    GS_FUNCTION_START_NODE(
+            "gs.function.startNode",
+            ReturnTypes.ARG0.andThen(
+                    (op, type) -> {
+                        GraphSchemaType edgeType = (GraphSchemaType) type;
+                        List<GraphLabelType.Entry> srcLabels =
+                                edgeType.getLabelType().getLabelsEntry().stream()
+                                        .map(
+                                                e ->
+                                                        new GraphLabelType.Entry()
+                                                                .label(e.getSrcLabel())
+                                                                .labelId(e.getSrcLabelId()))
+                                        .collect(Collectors.toList());
+                        return new GraphSchemaType(
+                                GraphOpt.Source.VERTEX,
+                                new GraphLabelType(srcLabels),
+                                ImmutableList.of());
+                    }),
+            GraphOperandTypes.family(GraphTypeFamily.EDGE),
+            GraphInferTypes.FIRST_KNOWN),
+
+    GS_FUNCTION_END_NODE(
+            "gs.function.endNode",
+            ReturnTypes.ARG0.andThen(
+                    (op, type) -> {
+                        GraphSchemaType edgeType = (GraphSchemaType) type;
+                        List<GraphLabelType.Entry> endLabels =
+                                edgeType.getLabelType().getLabelsEntry().stream()
+                                        .map(
+                                                e ->
+                                                        new GraphLabelType.Entry()
+                                                                .label(e.getDstLabel())
+                                                                .labelId(e.getDstLabelId()))
+                                        .collect(Collectors.toList());
+                        return new GraphSchemaType(
+                                GraphOpt.Source.VERTEX,
+                                new GraphLabelType(endLabels),
+                                ImmutableList.of());
+                    }),
+            GraphOperandTypes.family(GraphTypeFamily.EDGE),
+            GraphInferTypes.FIRST_KNOWN);
+
+    BuiltInFunction(
+            String signature,
+            SqlReturnTypeInference returnTypeInference,
+            SqlOperandTypeChecker operandTypeChecker,
+            SqlOperandTypeInference operandTypeInference) {
+        this.signature = signature;
+        this.returnTypeInference = returnTypeInference;
+        this.operandTypeChecker = operandTypeChecker;
+        this.operandTypeInference = operandTypeInference;
+    }
+
+    private final String signature;
+    private final SqlReturnTypeInference returnTypeInference;
+    private final SqlOperandTypeChecker operandTypeChecker;
+    private final SqlOperandTypeInference operandTypeInference;
+
+    public String getSignature() {
+        return this.signature;
+    }
+
+    public SqlReturnTypeInference getReturnTypeInference() {
+        return this.returnTypeInference;
+    }
+
+    public SqlOperandTypeChecker getOperandTypeChecker() {
+        return this.operandTypeChecker;
+    }
+
+    public SqlOperandTypeInference getOperandTypeInference() {
+        return this.operandTypeInference;
+    }
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/FunctionMeta.java

@@ -0,0 +1,78 @@
+/*
+ *
+ *  * Copyright 2020 Alibaba Group Holding Limited.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  * http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package com.alibaba.graphscope.common.ir.meta.function;
+
+import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta;
+
+import org.apache.calcite.sql.type.*;
+
+import java.util.stream.Collectors;
+
+public class FunctionMeta {
+    private final String signature;
+    private final SqlReturnTypeInference returnTypeInference;
+    private final SqlOperandTypeChecker operandTypeChecker;
+    private final SqlOperandTypeInference operandTypeInference;
+
+    public FunctionMeta(
+            String signature,
+            SqlReturnTypeInference returnTypeInference,
+            SqlOperandTypeChecker operandTypeChecker,
+            SqlOperandTypeInference operandTypeInference) {
+        this.signature = signature;
+        this.returnTypeInference = returnTypeInference;
+        this.operandTypeChecker = operandTypeChecker;
+        this.operandTypeInference = operandTypeInference;
+    }
+
+    public FunctionMeta(BuiltInFunction function) {
+        this(
+                function.getSignature(),
+                function.getReturnTypeInference(),
+                function.getOperandTypeChecker(),
+                function.getOperandTypeInference());
+    }
+
+    public FunctionMeta(StoredProcedureMeta meta) {
+        this(
+                meta.getName(),
+                ReturnTypes.explicit(meta.getReturnType().getFieldList().get(0).getType()),
+                GraphOperandTypes.metaTypeChecker(meta),
+                InferTypes.explicit(
+                        meta.getParameters().stream()
+                                .map(k -> k.getDataType())
+                                .collect(Collectors.toList())));
+    }
+
+    public String getSignature() {
+        return this.signature;
+    }
+
+    public SqlReturnTypeInference getReturnTypeInference() {
+        return this.returnTypeInference;
+    }
+
+    public SqlOperandTypeChecker getOperandTypeChecker() {
+        return this.operandTypeChecker;
+    }
+
+    public SqlOperandTypeInference getOperandTypeInference() {
+        return this.operandTypeInference;
+    }
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java

@@ -0,0 +1,119 @@
+/*
+ *
+ *  * Copyright 2020 Alibaba Group Holding Limited.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  * http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package com.alibaba.graphscope.common.ir.meta.function;
+
+import com.alibaba.graphscope.common.config.Configs;
+import com.alibaba.graphscope.common.config.GraphConfig;
+import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta;
+import com.google.common.collect.Maps;
+
+import org.apache.calcite.sql.type.GraphInferTypes;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+public class GraphFunctions {
+    public static final String FUNCTION_PREFIX = "gs.function.";
+    private final Map<String, FunctionMeta> functionMetaMap;
+
+    private static GraphFunctions instance;
+
+    public static GraphFunctions instance(Configs configs) {
+        if (instance == null) {
+            instance = new GraphFunctions(configs);
+        }
+        return instance;
+    }
+
+    private GraphFunctions(Configs configs) {
+        this.functionMetaMap = Maps.newHashMap();
+        this.registerBuiltInFunctions();
+        this.registerConfigFunctions(configs);
+    }
+
+    // initialize built-in functions
+    private void registerBuiltInFunctions() {
+        for (BuiltInFunction function : BuiltInFunction.values()) {
+            FunctionMeta meta = new FunctionMeta(function);
+            functionMetaMap.put(function.getSignature(), meta);
+        }
+    }
+
+    // initialize functions from configuration:
+    // 1. load the functions from the settings of 'graph.functions'
+    // 2. if the setting not found, load the default resource under the classpath
+    private void registerConfigFunctions(Configs configs) {
+        try (InputStream stream = loadConfigAsStream(configs)) {
+            Yaml yaml = new Yaml();
+            Map<String, Object> map = yaml.load(stream);
+            if (map != null) {
+                Object functions = map.get("graph_functions");
+                if (functions instanceof List) {
+                    for (Object function : (List) functions) {
+                        String functionYaml = yaml.dump(function);
+                        StoredProcedureMeta meta =
+                                StoredProcedureMeta.Deserializer.perform(
+                                        new ByteArrayInputStream(
+                                                functionYaml.getBytes(StandardCharsets.UTF_8)));
+                        functionMetaMap.put(meta.getName(), new FunctionMeta(meta));
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private InputStream loadConfigAsStream(Configs configs) throws Exception {
+        String functionConfig = GraphConfig.GRAPH_FUNCTIONS_URI.get(configs);
+        if (!functionConfig.isEmpty()) {
+            File file = new File(functionConfig);
+            if (file.exists()) {
+                return new FileInputStream(file);
+            }
+        }
+        return Thread.currentThread()
+                .getContextClassLoader()
+                .getResourceAsStream("conf/graph_functions.yaml");
+    }
+
+    public FunctionMeta getFunction(String functionName) {
+        FunctionMeta meta = functionMetaMap.get(functionName);
+        if (meta == null) {
+            // if not exist, create a new function meta with no constraints on operands and return
+            // types
+            meta =
+                    new FunctionMeta(
+                            functionName,
+                            ReturnTypes.explicit(SqlTypeName.ANY),
+                            OperandTypes.VARIADIC,
+                            GraphInferTypes.FIRST_KNOWN);
+        }
+        return meta;
+    }
+}