Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[VL] Spark mask function support #6271

Merged
merged 13 commits into from
Jul 30, 2024
Merged

[VL] Spark mask function support #6271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
33d38a3
mask function support
gaoyangxiaozhu Jun 28, 2024
a5d24bd
Merge branch 'apache:main' into gayangya/gayangya/mask_function
gaoyangxiaozhu Jul 1, 2024
29d9bd4
triiger CI
gaoyangxiaozhu Jul 1, 2024
a3bd595
Merge branch 'gayangya/gayangya/mask_function' of https://github.com/…
gaoyangxiaozhu Jul 1, 2024
e944b57
trigger ci
gaoyangxiaozhu Jul 1, 2024
cfa34b5
trigger ci
gaoyangxiaozhu Jul 1, 2024
8ad053b
Merge branch 'apache:main' into gayangya/gayangya/mask_function
gaoyangxiaozhu Jul 1, 2024
8fc3ece
[CI] Fix centos7 CI build error (#6298)
marin-ma Jul 1, 2024
85d20d7
Merge branch 'gayangya/gayangya/mask_function' of https://github.com/…
gaoyangxiaozhu Jul 2, 2024
553befd
Merge branch 'main' of https://github.com/gaoyangxiaozhu/gluten into …
gaoyangxiaozhu Jul 2, 2024
fd65eb0
Merge branch 'main' of https://github.com/gaoyangxiaozhu/gluten into …
gaoyangxiaozhu Jul 2, 2024
6f371c2
Merge branch 'main' of https://github.com/gaoyangxiaozhu/gluten into …
gaoyangxiaozhu Jul 30, 2024
5b4ffa5
small change
gaoyangxiaozhu Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,24 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateTest {
}
}

testWithSpecifiedSparkVersion("mask", Some("3.4")) {
gaoyangxiaozhu marked this conversation as resolved.
Show resolved Hide resolved
runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
runQueryAndCompare("SELECT mask(c_comment, 'Y') FROM customer limit 50") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y') FROM customer limit 50") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y', 'o') FROM customer limit 50") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y', 'o', '*') FROM customer limit 50") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}

test("bit_length") {
runQueryAndCompare(
"select bit_length(c_comment), bit_length(cast(c_comment as binary))" +
Expand Down
1 change: 1 addition & 0 deletions shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ object ExpressionNames {
final val LEVENSHTEIN = "levenshtein"
final val UNBASE64 = "unbase64"
final val BASE64 = "base64"
final val MASK = "mask"

// URL functions
final val PARSE_URL = "parse_url"
Expand Down
3 changes: 2 additions & 1 deletion shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ class Spark34Shims extends SparkShims {
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
Sig[RoundFloor](ExpressionNames.FLOOR),
Sig[RoundCeil](ExpressionNames.CEIL)
Sig[RoundCeil](ExpressionNames.CEIL),
Sig[Mask](ExpressionNames.MASK)
)
}

Expand Down
1 change: 1 addition & 0 deletions shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ class Spark35Shims extends SparkShims {
Sig[Csc](ExpressionNames.CSC),
Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[Mask](ExpressionNames.MASK),
Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
Sig[RoundFloor](ExpressionNames.FLOOR),
Sig[RoundCeil](ExpressionNames.CEIL)
Expand Down
Loading