diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index bd32a799c3ac..59833cc7ba3f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -815,6 +815,24 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + testWithSpecifiedSparkVersion("mask", Some("3.5")) { + runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + runQueryAndCompare("SELECT mask(c_comment, 'Y') FROM customer limit 50") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y') FROM customer limit 50") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y', 'o') FROM customer limit 50") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y', 'o', '*') FROM customer limit 50") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("bit_length") { runQueryAndCompare( "select bit_length(c_comment), bit_length(cast(c_comment as binary))" + diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 0adc1ce8ff61..d10e469e4d05 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,8 +16,8 @@ set -exu -VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_28 +VELOX_REPO=https://github.com/gaoyangxiaozhu/velox.git +VELOX_BRANCH=gayangya/mask_oap VELOX_HOME="" #Set on run gluten on HDFS diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 87b1b4e7539b..e791684ed442 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -130,6 +130,7 @@ object ExpressionNames { final val LEVENSHTEIN = "levenshteinDistance" final val UNBASE64 = "unbase64" final val BASE64 = "base64" + final val MASK = "mask" // URL functions final val PARSE_URL = "parse_url" diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 8ac8d323efd6..8b306442a65c 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -77,7 +77,8 @@ class Spark35Shims extends SparkShims { Sig[Sec](ExpressionNames.SEC), Sig[Csc](ExpressionNames.CSC), Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE), - Sig[Empty2Null](ExpressionNames.EMPTY2NULL) + Sig[Empty2Null](ExpressionNames.EMPTY2NULL), + Sig[Mask](ExpressionNames.MASK) ) }