From efce94e442b063868690500378f05c00cb8e5dc9 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 29 Mar 2024 15:26:03 +0800 Subject: [PATCH 1/4] Initial commit --- cpp/velox/operators/functions/RegistrationAllFunctions.cc | 3 --- cpp/velox/substrait/SubstraitParser.cc | 7 ------- 2 files changed, 10 deletions(-) diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index b827690d1cdf..cfe985b20a34 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -26,7 +26,6 @@ #include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" #include "velox/functions/prestosql/registration/RegistrationFunctions.h" #include "velox/functions/prestosql/window/WindowFunctionsRegistration.h" -#include "velox/functions/sparksql/Bitwise.h" #include "velox/functions/sparksql/Hash.h" #include "velox/functions/sparksql/Rand.h" #include "velox/functions/sparksql/Register.h" @@ -67,7 +66,6 @@ void registerFunctionOverwrite() { velox::exec::registerFunctionCallToSpecialForm( kRowConstructorWithAllNull, std::make_unique(kRowConstructorWithAllNull)); - velox::functions::sparksql::registerBitwiseFunctions("spark_"); velox::functions::registerBinaryIntegral({"check_add"}); velox::functions::registerBinaryIntegral({"check_subtract"}); velox::functions::registerBinaryIntegral({"check_multiply"}); @@ -79,7 +77,6 @@ void registerAllFunctions() { // The registration order matters. Spark sql functions are registered after // presto sql functions to overwrite the registration for same named // functions. - velox::functions::prestosql::registerAllScalarFunctions(); velox::functions::sparksql::registerFunctions(""); velox::aggregate::prestosql::registerAllAggregateFunctions( "", true /*registerCompanionFunctions*/, false /*onlyPrestoSignatures*/, true /*overwrite*/); diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index 5555ecfef954..890ddf9b63b7 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -391,13 +391,6 @@ std::unordered_map SubstraitParser::substraitVeloxFunc {"named_struct", "row_constructor"}, {"bit_or", "bitwise_or_agg"}, {"bit_and", "bitwise_and_agg"}, - {"bitwise_and", "spark_bitwise_and"}, - {"bitwise_not", "spark_bitwise_not"}, - {"bitwise_or", "spark_bitwise_or"}, - {"bitwise_xor", "spark_bitwise_xor"}, - // TODO: the below registry for rand functions can be removed - // after presto function registry is removed. - {"rand", "spark_rand"}, {"murmur3hash", "hash_with_seed"}, {"xxhash64", "xxhash64_with_seed"}, {"modulus", "remainder"}, From bda1bf657dc2117490411adbca9f40cd5914f04a Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 26 Jun 2024 17:48:25 +0800 Subject: [PATCH 2/4] Remove name mapping --- cpp/velox/substrait/SubstraitParser.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index 890ddf9b63b7..b842914ca933 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -396,11 +396,8 @@ std::unordered_map SubstraitParser::substraitVeloxFunc {"modulus", "remainder"}, {"date_format", "format_datetime"}, {"collect_set", "set_agg"}, - {"forall", "all_match"}, - {"exists", "any_match"}, {"negative", "unaryminus"}, - {"get_array_item", "get"}, - {"arrays_zip", "zip"}}; + {"get_array_item", "get"}}; const std::unordered_map SubstraitParser::typeMap_ = { {"bool", "BOOLEAN"}, From 056755123886e880600e772c624cb1cf69ea5fdb Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 26 Jun 2024 17:55:03 +0800 Subject: [PATCH 3/4] Try to register lacked ones --- .../operators/functions/RegistrationAllFunctions.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index cfe985b20a34..2b493abb9378 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -34,6 +34,14 @@ using namespace facebook; +namespace facebook::velox::functions { +void registerPrestoVectorFunctions() { + // Presto function. To be removed. + VELOX_REGISTER_VECTOR_FUNCTION(udf_arrays_overlap, "arrays_overlap"); + VELOX_REGISTER_VECTOR_FUNCTION(udf_transform_keys, "transform_keys"); + VELOX_REGISTER_VECTOR_FUNCTION(udf_transform_values, "transform_values"); +} +} // namespace facebook::velox::functions namespace gluten { namespace { void registerFunctionOverwrite() { @@ -70,6 +78,8 @@ void registerFunctionOverwrite() { velox::functions::registerBinaryIntegral({"check_subtract"}); velox::functions::registerBinaryIntegral({"check_multiply"}); velox::functions::registerBinaryIntegral({"check_divide"}); + + velox::functions::registerPrestoVectorFunctions(); } } // namespace From ebc6398a230d24760557e2d333fa85bf91daeb06 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 27 Jun 2024 08:57:18 +0800 Subject: [PATCH 4/4] Remove a comment --- cpp/velox/operators/functions/RegistrationAllFunctions.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index 2b493abb9378..638dbcccff0c 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -84,9 +84,6 @@ void registerFunctionOverwrite() { } // namespace void registerAllFunctions() { - // The registration order matters. Spark sql functions are registered after - // presto sql functions to overwrite the registration for same named - // functions. velox::functions::sparksql::registerFunctions(""); velox::aggregate::prestosql::registerAllAggregateFunctions( "", true /*registerCompanionFunctions*/, false /*onlyPrestoSignatures*/, true /*overwrite*/);