Skip to content

Commit

Permalink
Spark: Support singular form of years, months, days, and hours functions
Browse files Browse the repository at this point in the history
  • Loading branch information
@wypoon
wypoon committed Jan 27, 2025
1 parent ce2af52 commit 87c408b
Show file tree
Hide file tree
Showing 5 changed files with 46 additions and 8 deletions.
21 changes: 13 additions & 8 deletions spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/functions/SparkFunctions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,19 @@ public class SparkFunctions {
private SparkFunctions() {}

private static final Map<String, UnboundFunction> FUNCTIONS =
ImmutableMap.of(
"iceberg_version", new IcebergVersionFunction(),
"years", new YearsFunction(),
"months", new MonthsFunction(),
"days", new DaysFunction(),
"hours", new HoursFunction(),
"bucket", new BucketFunction(),
"truncate", new TruncateFunction());
new ImmutableMap.Builder<String, UnboundFunction>()
.put("iceberg_version", new IcebergVersionFunction())
.put("years", new YearsFunction())
.put("year", new YearsFunction())
.put("months", new MonthsFunction())
.put("month", new MonthsFunction())
.put("days", new DaysFunction())
.put("day", new DaysFunction())
.put("hours", new HoursFunction())
.put("hour", new HoursFunction())
.put("bucket", new BucketFunction())
.put("truncate", new TruncateFunction())
.build();

private static final Map<Class<?>, UnboundFunction> CLASS_TO_FUNCTIONS =
ImmutableMap.of(
Expand Down
9 changes: 9 additions & 0 deletions spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkDaysFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ public void testDates() {
assertThat(scalarSql("SELECT system.days(date('2017-12-01'))"))
.as("Expected to produce 2017-12-01")
.isEqualTo(Date.valueOf("2017-12-01"));
assertThat(scalarSql("SELECT system.day(date('2017-12-01'))"))
.as("Expected to produce 2017-12-01")
.isEqualTo(Date.valueOf("2017-12-01"));
assertThat(scalarSql("SELECT system.days(date('1970-01-01'))"))
.as("Expected to produce 1970-01-01")
.isEqualTo(Date.valueOf("1970-01-01"));
Expand All @@ -53,6 +56,9 @@ public void testTimestamps() {
assertThat(scalarSql("SELECT system.days(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 2017-12-01")
.isEqualTo(Date.valueOf("2017-12-01"));
assertThat(scalarSql("SELECT system.day(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 2017-12-01")
.isEqualTo(Date.valueOf("2017-12-01"));
assertThat(scalarSql("SELECT system.days(TIMESTAMP '1970-01-01 00:00:01.000001 UTC+00:00')"))
.as("Expected to produce 1970-01-01")
.isEqualTo(Date.valueOf("1970-01-01"));
Expand All @@ -67,6 +73,9 @@ public void testTimestampNtz() {
assertThat(scalarSql("SELECT system.days(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 2017-12-01")
.isEqualTo(Date.valueOf("2017-12-01"));
assertThat(scalarSql("SELECT system.day(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 2017-12-01")
.isEqualTo(Date.valueOf("2017-12-01"));
assertThat(scalarSql("SELECT system.days(TIMESTAMP_NTZ '1970-01-01 00:00:01.000001 UTC')"))
.as("Expected to produce 1970-01-01")
.isEqualTo(Date.valueOf("1970-01-01"));
Expand Down
6 changes: 6 additions & 0 deletions spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkHoursFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ public void testTimestamps() {
assertThat(scalarSql("SELECT system.hours(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 17501 * 24 + 10")
.isEqualTo(420034);
assertThat(scalarSql("SELECT system.hour(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 17501 * 24 + 10")
.isEqualTo(420034);
assertThat(scalarSql("SELECT system.hours(TIMESTAMP '1970-01-01 00:00:01.000001 UTC+00:00')"))
.as("Expected to produce 0 * 24 + 0 = 0")
.isEqualTo(0);
Expand All @@ -52,6 +55,9 @@ public void testTimestampsNtz() {
assertThat(scalarSql("SELECT system.hours(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 17501 * 24 + 10")
.isEqualTo(420034);
assertThat(scalarSql("SELECT system.hour(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 17501 * 24 + 10")
.isEqualTo(420034);
assertThat(scalarSql("SELECT system.hours(TIMESTAMP_NTZ '1970-01-01 00:00:01.000001 UTC')"))
.as("Expected to produce 0 * 24 + 0 = 0")
.isEqualTo(0);
Expand Down
9 changes: 9 additions & 0 deletions spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkMonthsFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ public void testDates() {
assertThat(scalarSql("SELECT system.months(date('2017-12-01'))"))
.as("Expected to produce 47 * 12 + 11 = 575")
.isEqualTo(575);
assertThat(scalarSql("SELECT system.month(date('2017-12-01'))"))
.as("Expected to produce 47 * 12 + 11 = 575")
.isEqualTo(575);
assertThat(scalarSql("SELECT system.months(date('1970-01-01'))"))
.as("Expected to produce 0 * 12 + 0 = 0")
.isEqualTo(0);
Expand All @@ -53,6 +56,9 @@ public void testTimestamps() {
assertThat(scalarSql("SELECT system.months(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 47 * 12 + 11 = 575")
.isEqualTo(575);
assertThat(scalarSql("SELECT system.month(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 47 * 12 + 11 = 575")
.isEqualTo(575);
assertThat(scalarSql("SELECT system.months(TIMESTAMP '1970-01-01 00:00:01.000001 UTC+00:00')"))
.as("Expected to produce 0 * 12 + 0 = 0")
.isEqualTo(0);
Expand All @@ -67,6 +73,9 @@ public void testTimestampNtz() {
assertThat(scalarSql("SELECT system.months(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 47 * 12 + 11 = 575")
.isEqualTo(575);
assertThat(scalarSql("SELECT system.month(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 47 * 12 + 11 = 575")
.isEqualTo(575);
assertThat(scalarSql("SELECT system.months(TIMESTAMP_NTZ '1970-01-01 00:00:01.000001 UTC')"))
.as("Expected to produce 0 * 12 + 0 = 0")
.isEqualTo(0);
Expand Down
9 changes: 9 additions & 0 deletions spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkYearsFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ public void testDates() {
assertThat(scalarSql("SELECT system.years(date('2017-12-01'))"))
.as("Expected to produce 2017 - 1970 = 47")
.isEqualTo(47);
assertThat(scalarSql("SELECT system.year(date('2017-12-01'))"))
.as("Expected to produce 2017 - 1970 = 47")
.isEqualTo(47);
assertThat(scalarSql("SELECT system.years(date('1970-01-01'))"))
.as("Expected to produce 1970 - 1970 = 0")
.isEqualTo(0);
Expand All @@ -53,6 +56,9 @@ public void testTimestamps() {
assertThat(scalarSql("SELECT system.years(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 2017 - 1970 = 47")
.isEqualTo(47);
assertThat(scalarSql("SELECT system.year(TIMESTAMP '2017-12-01 10:12:55.038194 UTC+00:00')"))
.as("Expected to produce 2017 - 1970 = 47")
.isEqualTo(47);
assertThat(scalarSql("SELECT system.years(TIMESTAMP '1970-01-01 00:00:01.000001 UTC+00:00')"))
.as("Expected to produce 1970 - 1970 = 0")
.isEqualTo(0);
Expand All @@ -67,6 +73,9 @@ public void testTimestampNtz() {
assertThat(scalarSql("SELECT system.years(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 2017 - 1970 = 47")
.isEqualTo(47);
assertThat(scalarSql("SELECT system.year(TIMESTAMP_NTZ '2017-12-01 10:12:55.038194 UTC')"))
.as("Expected to produce 2017 - 1970 = 47")
.isEqualTo(47);
assertThat(scalarSql("SELECT system.years(TIMESTAMP_NTZ '1970-01-01 00:00:01.000001 UTC')"))
.as("Expected to produce 1970 - 1970 = 0")
.isEqualTo(0);
Expand Down

0 comments on commit 87c408b

Please sign in to comment.