[GLUTEN-6345][CH] Deprecate SCALAR_FUNCTIONS in SerializedPlanParser (#…

…6347) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) Fixes: #6345 Following map in SerializedPlanParser is removed. All functions are parsed by FunctionParser now. static const std::map<std::string, std::string> SCALAR_FUNCTIONS One could register a simple function in Parser/scalar_function_parser/CommonScalarFunctionParser.cpp REGISTER_COMMON_SCALAR_FUNCTION_PARSER(MothsBetween, months_between, sparkMonthsBetween); How was this patch tested? (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) unit tests (If this patch involves UI changes, please attach a screenshot; otherwise, remove this)
apache · Jul 10, 2024 · b175365 · b175365
1 parent ca35e47
commit b175365
Show file tree

Hide file tree

Showing 78 changed files with 1,247 additions and 698 deletions.
diff --git a/cpp-ch/local-engine/Parser/AggregateFunctionParser.cpp b/cpp-ch/local-engine/Parser/AggregateFunctionParser.cpp
@@ -39,7 +39,6 @@ namespace local_engine
 
 DB::ActionsDAG::NodeRawConstPtrs AggregateFunctionParser::parseFunctionArguments(
     const CommonFunctionInfo & func_info,
-    const String & /*ch_func_name*/,
     DB::ActionsDAGPtr & actions_dag) const
 {
     DB::ActionsDAG::NodeRawConstPtrs collected_args;

diff --git a/cpp-ch/local-engine/Parser/AggregateFunctionParser.h b/cpp-ch/local-engine/Parser/AggregateFunctionParser.h
@@ -97,12 +97,7 @@ class AggregateFunctionParser
 
     /// Do some preprojections for the function arguments, and return the necessary arguments for the CH function.
     virtual DB::ActionsDAG::NodeRawConstPtrs
-    parseFunctionArguments(const CommonFunctionInfo & func_info, const String & ch_func_name, DB::ActionsDAGPtr & actions_dag) const;
-
-    DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments(const CommonFunctionInfo & func_info, DB::ActionsDAGPtr & actions_dag) const
-    {
-        return parseFunctionArguments(func_info, getCHFunctionName(func_info), actions_dag);
-    }
+    parseFunctionArguments(const CommonFunctionInfo & func_info, DB::ActionsDAGPtr & actions_dag) const;
 
     // `PartialMerge` is applied on the merging stages.
     // `If` is applied when the aggreate function has a filter. This should only happen on the 1st stage.

diff --git a/cpp-ch/local-engine/Parser/FunctionParser.cpp b/cpp-ch/local-engine/Parser/FunctionParser.cpp
@@ -20,6 +20,7 @@
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/IDataType.h>
 #include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
 #include <Parser/TypeParser.h>
 #include <Common/CHUtil.h>
 
@@ -39,24 +40,18 @@ using namespace DB;
 
 String FunctionParser::getCHFunctionName(const substrait::Expression_ScalarFunction & substrait_func) const
 {
-    auto func_signature = plan_parser->function_mapping.at(std::to_string(substrait_func.function_reference()));
-    auto pos = func_signature.find(':');
-    auto func_name = func_signature.substr(0, pos);
-
-    auto it = SCALAR_FUNCTIONS.find(func_name);
-    if (it == SCALAR_FUNCTIONS.end())
-        throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unsupported substrait function: {}", func_name);
-    return it->second;
+    // no meaning
+    /// There is no any simple equivalent ch function.
+    return "";
 }
 
 ActionsDAG::NodeRawConstPtrs FunctionParser::parseFunctionArguments(
-    const substrait::Expression_ScalarFunction & substrait_func, const String & ch_func_name, ActionsDAGPtr & actions_dag) const
+    const substrait::Expression_ScalarFunction & substrait_func, ActionsDAGPtr & actions_dag) const
 {
     ActionsDAG::NodeRawConstPtrs parsed_args;
     const auto & args = substrait_func.arguments();
     parsed_args.reserve(args.size());
-    for (const auto & arg : args)
-        plan_parser->parseFunctionArgument(actions_dag, parsed_args, ch_func_name, arg);
+    plan_parser->parseFunctionArguments(actions_dag, parsed_args, substrait_func);
     return parsed_args;
 }
 
@@ -66,7 +61,7 @@ const ActionsDAG::Node *
 FunctionParser::parse(const substrait::Expression_ScalarFunction & substrait_func, ActionsDAGPtr & actions_dag) const
 {
     auto ch_func_name = getCHFunctionName(substrait_func);
-    auto parsed_args = parseFunctionArguments(substrait_func, ch_func_name, actions_dag);
+    auto parsed_args = parseFunctionArguments(substrait_func, actions_dag);
     const auto * func_node = toFunctionNode(actions_dag, ch_func_name, parsed_args);
     return convertNodeTypeIfNeeded(substrait_func, func_node, actions_dag);
 }
@@ -76,13 +71,30 @@ const ActionsDAG::Node * FunctionParser::convertNodeTypeIfNeeded(
 {
     const auto & output_type = substrait_func.output_type();
     if (!TypeParser::isTypeMatched(output_type, func_node->result_type))
-        return ActionsDAGUtil::convertNodeType(
-            actions_dag,
-            func_node,
-            // as stated in isTypeMatched， currently we don't change nullability of the result type
-            func_node->result_type->isNullable() ? local_engine::wrapNullableType(true, TypeParser::parseType(output_type))->getName()
-                                                 : DB::removeNullable(TypeParser::parseType(output_type))->getName(),
-            func_node->result_name);
+    {
+        auto result_type = TypeParser::parseType(substrait_func.output_type());
+        if (DB::isDecimalOrNullableDecimal(result_type))
+        {
+            return ActionsDAGUtil::convertNodeType(
+                actions_dag,
+                func_node,
+                // as stated in isTypeMatched， currently we don't change nullability of the result type
+                func_node->result_type->isNullable() ? local_engine::wrapNullableType(true, result_type)->getName()
+                                                         : local_engine::removeNullable(result_type)->getName(),
+                func_node->result_name,
+                CastType::accurateOrNull);
+        }
+        else
+        {
+            return ActionsDAGUtil::convertNodeType(
+                actions_dag,
+                func_node,
+                // as stated in isTypeMatched， currently we don't change nullability of the result type
+                func_node->result_type->isNullable() ? local_engine::wrapNullableType(true, TypeParser::parseType(output_type))->getName()
+                                                     : DB::removeNullable(TypeParser::parseType(output_type))->getName(),
+                func_node->result_name);
+        }
+    }
     else
         return func_node;
 }

diff --git a/cpp-ch/local-engine/Parser/FunctionParser.h b/cpp-ch/local-engine/Parser/FunctionParser.h
@@ -51,10 +51,16 @@ class FunctionParser
 
     virtual String getCHFunctionName(const substrait::Expression_ScalarFunction & substrait_func) const;
 protected:
-
+    /// Deprecated method
+    virtual DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments(
+        const substrait::Expression_ScalarFunction & substrait_func,
+        const String & /*function_name*/,
+        DB::ActionsDAGPtr & actions_dag) const
+    {
+        return parseFunctionArguments(substrait_func, actions_dag);
+    }
     virtual DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments(
         const substrait::Expression_ScalarFunction & substrait_func,
-        const String & ch_func_name,
         DB::ActionsDAGPtr & actions_dag) const;
 
     virtual const DB::ActionsDAG::Node * convertNodeTypeIfNeeded(
@@ -84,14 +90,19 @@ class FunctionParser
         return &action_dag->addFunction(function_builder, args, result_name);
     }
 
+    const ActionsDAG::Node *
+        parseFunctionWithDAG(const substrait::Expression & rel, std::string & result_name, DB::ActionsDAGPtr actions_dag, bool keep_result = false) const
+    {
+        return plan_parser->parseFunctionWithDAG(rel, result_name, actions_dag, keep_result);
+    }
     const DB::ActionsDAG::Node * parseExpression(DB::ActionsDAGPtr actions_dag, const substrait::Expression & rel) const
     {
         return plan_parser->parseExpression(actions_dag, rel);
     }
 
     std::pair<DataTypePtr, Field> parseLiteral(const substrait::Expression_Literal & literal) const { return plan_parser->parseLiteral(literal); }
 
-    SerializedPlanParser * plan_parser;
+    mutable SerializedPlanParser * plan_parser;
 };
 
 using FunctionParserPtr = std::shared_ptr<FunctionParser>;

diff --git a/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp b/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp
@@ -383,13 +383,7 @@ void MergeTreeRelParser::collectColumns(const substrait::Expression & rel, NameS
 String MergeTreeRelParser::getCHFunctionName(const substrait::Expression_ScalarFunction & substrait_func)
 {
     auto func_signature = getPlanParser()->function_mapping.at(std::to_string(substrait_func.function_reference()));
-    auto pos = func_signature.find(':');
-    auto func_name = func_signature.substr(0, pos);
-
-    auto it = SCALAR_FUNCTIONS.find(func_name);
-    if (it == SCALAR_FUNCTIONS.end())
-        throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unsupported substrait function on mergetree prewhere parser: {}", func_name);
-    return it->second;
+    return getPlanParser()->getFunctionName(func_signature, substrait_func);
 }