Skip to content

Commit

Permalink
[GLUTEN-6809][CH] Support function unix_seconds/unix_date/unix_micros…
Browse files Browse the repository at this point in the history 
…/unix_millis (apache#7094)

* support function unix_seconds

* support function unix_date

* support function unix_millis and unix_micros

* add uts

* add uts
  • Loading branch information
@taiyang-li
taiyang-li authored and shamirchen committed Oct 14, 2024
1 parent 89177aa commit e5107c7
Show file tree
Hide file tree
Showing 7 changed files with 25 additions and 19 deletions.
4 changes: 0 additions & 4 deletions backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,6 @@ object CHExpressionUtil {
ARRAY_REMOVE -> DefaultValidator(),
ARRAYS_ZIP -> DefaultValidator(),
DATE_FROM_UNIX_DATE -> DefaultValidator(),
UNIX_DATE -> DefaultValidator(),
UNIX_SECONDS -> DefaultValidator(),
MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
SPARK_PARTITION_ID -> DefaultValidator(),
URL_DECODE -> DefaultValidator(),
Expand All @@ -218,8 +216,6 @@ object CHExpressionUtil {
REGR_SXY -> DefaultValidator(),
TO_UTC_TIMESTAMP -> UtcTimestampValidator(),
FROM_UTC_TIMESTAMP -> UtcTimestampValidator(),
UNIX_MILLIS -> DefaultValidator(),
UNIX_MICROS -> DefaultValidator(),
TIMESTAMP_MILLIS -> DefaultValidator(),
TIMESTAMP_MICROS -> DefaultValidator(),
STACK -> DefaultValidator(),
Expand Down
21 changes: 19 additions & 2 deletions ...s-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,9 +757,26 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS
}

test("test function array_except") {
val sql =
"""
|SELECT array_except(array(id, id+1, id+2), array(id+2, id+3))
|FROM RANGE(10)
|""".stripMargin
runQueryAndCompare(sql)(checkGlutenOperatorMatch[ProjectExecTransformer])
}

test("test functions unix_seconds/unix_date/unix_millis/unix_micros") {
val sql = """
|SELECT array_except(array(id, id+1, id+2), array(id+2, id+3))
|FROM RANGE(10)
|SELECT
| id,
| unix_seconds(cast(concat('2024-09-03 17:23:1',
| cast(id as string)) as timestamp)),
| unix_date(cast(concat('2024-09-1', cast(id as string)) as date)),
| unix_millis(cast(concat('2024-09-03 17:23:10.11',
| cast(id as string)) as timestamp)),
| unix_micros(cast(concat('2024-09-03 17:23:10.12345',
| cast(id as string)) as timestamp))
|FROM range(10)
|""".stripMargin
runQueryAndCompare(sql)(checkGlutenOperatorMatch[ProjectExecTransformer])
}
Expand Down
7 changes: 6 additions & 1 deletion cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(LT, lt, less);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(And, and, and);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Or, or, or);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Equal, equal, equals);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Not, not, not);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Not, not, not );
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Xor, xor, xor);

REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Cast, cast, CAST);
Expand Down Expand Up @@ -143,6 +143,7 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(In, in, in);

REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Coalesce, coalesce, coalesce);

// date time functions
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(FromUnixtime, from_unixtime, fromUnixTimestampInJodaSyntax);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateAdd, date_add, addDays);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateSub, date_sub, subtractDays);
Expand All @@ -153,6 +154,10 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateTrunc, date_trunc, dateTrunc);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(FloorDatetime, floor_datetime, dateTrunc);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Floor, floor, sparkFloor);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(MothsBetween, months_between, sparkMonthsBetween);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixSeconds, unix_seconds, toUnixTimestamp);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixDate, unix_date, toInt32);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMillis, unix_millis, toUnixTimestamp64Milli);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMicros, unix_micros, toUnixTimestamp64Micro);

// array functions
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Array, array, array);
Expand Down
3 changes: 0 additions & 3 deletions ...ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,10 +739,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is missing")
.exclude("DATE_FROM_UNIX_DATE")
.exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
.exclude("UNIX_MILLIS")
.exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
Expand Down
3 changes: 0 additions & 3 deletions ...ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -768,10 +768,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is missing")
.exclude("DATE_FROM_UNIX_DATE")
.exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
.exclude("UNIX_MILLIS")
.exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
Expand Down
3 changes: 0 additions & 3 deletions ...ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,10 +656,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is missing")
.exclude("DATE_FROM_UNIX_DATE")
.exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
.exclude("UNIX_MILLIS")
.exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
Expand Down
3 changes: 0 additions & 3 deletions ...ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,10 +656,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is missing")
.exclude("DATE_FROM_UNIX_DATE")
.exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
.exclude("UNIX_MILLIS")
.exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
Expand Down

0 comments on commit e5107c7

Please sign in to comment.