Skip to content

Commit

Permalink
Add array_remove spark function (facebookincubator#9215)
Browse files Browse the repository at this point in the history
Summary:
Spark is the same as the `array_remove` function of Presto.
Spark `array_remove` doc: https://spark.apache.org/docs/latest/api/sql/index.html#array_remove

facebookincubator#9214

Pull Request resolved: facebookincubator#9215

Reviewed By: Yuhta

Differential Revision: D55411381

Pulled By: mbasmanova

fbshipit-source-id: aaccb8cd2421da67bb98e012fdd8bd6dcb366b72
  • Loading branch information
dcoliversun authored and facebook-github-bot committed Mar 29, 2024
1 parent 7821329 commit cb58cba
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 0 deletions.
12 changes: 12 additions & 0 deletions velox/docs/functions/spark/array.rst
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,18 @@ Array Functions
SELECT array_min(ARRAY [4.0, float('nan')]); -- 4.0
SELECT array_min(ARRAY [NULL, float('nan')]); -- NaN

.. function:: array_remove(x, element) -> array

Remove all elements that equal ``element`` from array ``x``. Returns NULL as result if ``element`` is NULL.
If array ``x`` is empty array, returns empty array. If all elements in array ``x`` are NULL but ``element`` is not NULL,
returns array ``x``. ::

SELECT array_remove(ARRAY [1, 2, 3], 3); -- [1, 2]
SELECT array_remove(ARRAY [2, 1, NULL], 1); -- [2, NULL]
SELECT array_remove(ARRAY [1, 2, NULL], NULL); -- NULL
SELECT array_remove(ARRAY [], 1); -- []
SELECT array_remove(ARRAY [NULL, NULL], -1); -- [NULL, NULL]

.. spark:function:: array_repeat(element, count) -> array(E)
Returns an array containing ``element`` ``count`` times. If ``count`` is negative or zero,
Expand Down
28 changes: 28 additions & 0 deletions velox/functions/sparksql/Register.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "velox/functions/lib/Re2Functions.h"
#include "velox/functions/lib/RegistrationHelpers.h"
#include "velox/functions/lib/Repeat.h"
#include "velox/functions/prestosql/ArrayFunctions.h"
#include "velox/functions/prestosql/DateTimeFunctions.h"
#include "velox/functions/prestosql/JsonFunctions.h"
#include "velox/functions/prestosql/StringFunctions.h"
Expand Down Expand Up @@ -52,6 +53,32 @@ extern void registerElementAtFunction(
const std::string& name,
bool enableCaching);

template <typename T>
inline void registerArrayRemoveFunctions(const std::string& prefix) {
registerFunction<ArrayRemoveFunction, Array<T>, Array<T>, T>(
{prefix + "array_remove"});
}

inline void registerArrayRemoveFunctions(const std::string& prefix) {
registerArrayRemoveFunctions<int8_t>(prefix);
registerArrayRemoveFunctions<int16_t>(prefix);
registerArrayRemoveFunctions<int32_t>(prefix);
registerArrayRemoveFunctions<int64_t>(prefix);
registerArrayRemoveFunctions<int128_t>(prefix);
registerArrayRemoveFunctions<float>(prefix);
registerArrayRemoveFunctions<double>(prefix);
registerArrayRemoveFunctions<bool>(prefix);
registerArrayRemoveFunctions<Timestamp>(prefix);
registerArrayRemoveFunctions<Date>(prefix);
registerArrayRemoveFunctions<Varbinary>(prefix);
registerArrayRemoveFunctions<Generic<T1>>(prefix);
registerFunction<
ArrayRemoveFunctionString,
Array<Varchar>,
Array<Varchar>,
Varchar>({prefix + "array_remove"});
}

static void workAroundRegistrationMacro(const std::string& prefix) {
// VELOX_REGISTER_VECTOR_FUNCTION must be invoked in the same namespace as the
// vector function definition.
Expand Down Expand Up @@ -82,6 +109,7 @@ static void workAroundRegistrationMacro(const std::string& prefix) {
VELOX_REGISTER_VECTOR_FUNCTION(udf_not, prefix + "not");
registerIsNullFunction(prefix + "isnull");
registerIsNotNullFunction(prefix + "isnotnull");
registerArrayRemoveFunctions(prefix);
}

namespace sparksql {
Expand Down

0 comments on commit cb58cba

Please sign in to comment.