diff --git a/velox/docs/functions/spark/datetime.rst b/velox/docs/functions/spark/datetime.rst index d7ed56e5fc51..6c6de79332ff 100644 --- a/velox/docs/functions/spark/datetime.rst +++ b/velox/docs/functions/spark/datetime.rst @@ -221,6 +221,20 @@ These functions support TIMESTAMP and DATE input types. SELECT second('2009-07-30 12:58:59'); -- 59 +.. spark:function:: timestamp_micros(x) -> timestamp + + Returns timestamp from the number of microseconds since UTC epoch. + Supported types are: TINYINT, SMALLINT, INTEGER and BIGINT.:: + + SELECT timestamp_micros(1230219000123123); -- '2008-12-25 15:30:00.123123' + +.. spark:function:: timestamp_millis(x) -> timestamp + + Returns timestamp from the number of milliseconds since UTC epoch. + Supported types are: TINYINT, SMALLINT, INTEGER and BIGINT.:: + + SELECT timestamp_millis(1230219000123); -- '2008-12-25 15:30:00.123' + .. spark:function:: to_unix_timestamp(string) -> integer Alias for ``unix_timestamp(string) -> integer``. @@ -244,6 +258,15 @@ These functions support TIMESTAMP and DATE input types. SELECT unix_date('1970-01-02'); -- '1' SELECT unix_date('1969-12-31'); -- '-1' +.. spark:function:: unix_micros(timestamp) -> bigint + + Returns the number of microseconds since 1970-01-01 00:00:00 UTC. + +.. spark:function:: unix_millis(timestamp) -> bigint + + Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates + higher levels of precision. + .. spark:function:: unix_timestamp() -> integer Returns the current UNIX timestamp in seconds. diff --git a/velox/functions/sparksql/DateTimeFunctions.h b/velox/functions/sparksql/DateTimeFunctions.h index 0ba287089e4a..a8f1630a5001 100644 --- a/velox/functions/sparksql/DateTimeFunctions.h +++ b/velox/functions/sparksql/DateTimeFunctions.h @@ -731,4 +731,51 @@ struct MakeYMIntervalFunction { result = totalMonths; } }; + +template +struct TimestampToMicrosFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + + FOLLY_ALWAYS_INLINE void call( + int64_t& result, + const arg_type& timestamp) { + result = timestamp.toMicros(); + } +}; + +template +struct MicrosToTimestampFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + + template + FOLLY_ALWAYS_INLINE void call( + out_type& result, + const TInput micros) { + result = Timestamp::fromMicros(micros); + } +}; + +template +struct TimestampToMillisFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + + FOLLY_ALWAYS_INLINE void call( + int64_t& result, + const arg_type& timestamp) { + result = timestamp.toMillis(); + } +}; + +template +struct MillisToTimestampFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + + template + FOLLY_ALWAYS_INLINE void call( + out_type& result, + const TInput millis) { + result = Timestamp::fromMillis(millis); + } +}; + } // namespace facebook::velox::functions::sparksql diff --git a/velox/functions/sparksql/Register.cpp b/velox/functions/sparksql/Register.cpp index 72a8ca527568..190878555d3a 100644 --- a/velox/functions/sparksql/Register.cpp +++ b/velox/functions/sparksql/Register.cpp @@ -388,6 +388,27 @@ void registerFunctions(const std::string& prefix) { VELOX_REGISTER_VECTOR_FUNCTION(udf_make_timestamp, prefix + "make_timestamp"); + registerFunction( + {prefix + "unix_micros"}); + registerFunction( + {prefix + "timestamp_micros"}); + registerFunction( + {prefix + "timestamp_micros"}); + registerFunction( + {prefix + "timestamp_micros"}); + registerFunction( + {prefix + "timestamp_micros"}); + registerFunction( + {prefix + "unix_millis"}); + registerFunction( + {prefix + "timestamp_millis"}); + registerFunction( + {prefix + "timestamp_millis"}); + registerFunction( + {prefix + "timestamp_millis"}); + registerFunction( + {prefix + "timestamp_millis"}); + // Register bloom filter function registerFunction( {prefix + "might_contain"}); diff --git a/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp b/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp index 383f8bd48d64..a7ed87c283f9 100644 --- a/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp +++ b/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp @@ -40,6 +40,8 @@ class DateTimeFunctionsTest : public SparkFunctionBaseTest { static constexpr int16_t kMaxSmallint = std::numeric_limits::max(); static constexpr int8_t kMinTinyint = std::numeric_limits::min(); static constexpr int8_t kMaxTinyint = std::numeric_limits::max(); + static constexpr int64_t kMinLong = std::numeric_limits::min(); + static constexpr int64_t kMaxLong = std::numeric_limits::max(); protected: void setQueryTimeZone(const std::string& timeZone) { @@ -968,5 +970,116 @@ TEST_F(DateTimeFunctionsTest, makeYMInterval) { fromYear(-178956971), "Integer overflow in make_ym_interval(-178956971)"); } +TEST_F(DateTimeFunctionsTest, microsToTimestamp) { + const auto microsToTimestamp = [&](int64_t micros) { + return evaluateOnce("timestamp_micros(c0)", micros); + }; + EXPECT_EQ( + microsToTimestamp(1000000), + util::fromTimestampString("1970-01-01 00:00:01")); + EXPECT_EQ( + microsToTimestamp(1230219000123123), + util::fromTimestampString("2008-12-25 15:30:00.123123")); + + EXPECT_EQ( + microsToTimestamp(kMaxTinyint), + util::fromTimestampString("1970-01-01 00:00:00.000127")); + EXPECT_EQ( + microsToTimestamp(kMinTinyint), + util::fromTimestampString("1969-12-31 23:59:59.999872")); + EXPECT_EQ( + microsToTimestamp(kMaxSmallint), + util::fromTimestampString("1970-01-01 00:00:00.032767")); + EXPECT_EQ( + microsToTimestamp(kMinSmallint), + util::fromTimestampString("1969-12-31 23:59:59.967232")); + EXPECT_EQ( + microsToTimestamp(kMax), + util::fromTimestampString("1970-01-01 00:35:47.483647")); + EXPECT_EQ( + microsToTimestamp(kMin), + util::fromTimestampString("1969-12-31 23:24:12.516352")); + EXPECT_EQ( + microsToTimestamp(kMaxLong), + util::fromTimestampString("294247-01-10 04:00:54.775807")); + EXPECT_EQ( + microsToTimestamp(kMinLong), + util::fromTimestampString("-290308-12-21 19:59:05.224192")); +} + +TEST_F(DateTimeFunctionsTest, millisToTimestamp) { + const auto millisToTimestamp = [&](int64_t millis) { + return evaluateOnce("timestamp_millis(c0)", millis); + }; + EXPECT_EQ( + millisToTimestamp(1000), + util::fromTimestampString("1970-01-01 00:00:01")); + EXPECT_EQ( + millisToTimestamp(1230219000123), + util::fromTimestampString("2008-12-25 15:30:00.123")); + + EXPECT_EQ( + millisToTimestamp(kMaxTinyint), + util::fromTimestampString("1970-01-01 00:00:00.127")); + EXPECT_EQ( + millisToTimestamp(kMinTinyint), + util::fromTimestampString("1969-12-31 23:59:59.872")); + EXPECT_EQ( + millisToTimestamp(kMaxSmallint), + util::fromTimestampString("1970-01-01 00:00:32.767")); + EXPECT_EQ( + millisToTimestamp(kMinSmallint), + util::fromTimestampString("1969-12-31 23:59:27.232")); + EXPECT_EQ( + millisToTimestamp(kMax), + util::fromTimestampString("1970-01-25 20:31:23.647")); + EXPECT_EQ( + millisToTimestamp(kMin), + util::fromTimestampString("1969-12-07 03:28:36.352")); + EXPECT_EQ( + millisToTimestamp(kMaxLong), + util::fromTimestampString("292278994-08-17T07:12:55.807")); + EXPECT_EQ( + millisToTimestamp(kMinLong), + util::fromTimestampString("-292275055-05-16T16:47:04.192")); +} + +TEST_F(DateTimeFunctionsTest, timestampToMicros) { + const auto timestampToMicros = [&](const StringView time) { + return evaluateOnce( + "unix_micros(c0)", util::fromTimestampString(time)); + }; + EXPECT_EQ(timestampToMicros("1970-01-01 00:00:01"), 1000000); + EXPECT_EQ(timestampToMicros("2008-12-25 15:30:00.123123"), 1230219000123123); + + EXPECT_EQ(timestampToMicros("1970-01-01 00:00:00.000127"), kMaxTinyint); + EXPECT_EQ(timestampToMicros("1969-12-31 23:59:59.999872"), kMinTinyint); + EXPECT_EQ(timestampToMicros("1970-01-01 00:00:00.032767"), kMaxSmallint); + EXPECT_EQ(timestampToMicros("1969-12-31 23:59:59.967232"), kMinSmallint); + EXPECT_EQ(timestampToMicros("1970-01-01 00:35:47.483647"), kMax); + EXPECT_EQ(timestampToMicros("1969-12-31 23:24:12.516352"), kMin); + EXPECT_EQ(timestampToMicros("294247-01-10 04:00:54.775807"), kMaxLong); + EXPECT_EQ( + timestampToMicros("-290308-12-21 19:59:06.224192"), kMinLong + 1000000); +} + +TEST_F(DateTimeFunctionsTest, timestampToMillis) { + const auto timestampToMillis = [&](const StringView time) { + return evaluateOnce( + "unix_millis(c0)", util::fromTimestampString(time)); + }; + EXPECT_EQ(timestampToMillis("1970-01-01 00:00:01"), 1000); + EXPECT_EQ(timestampToMillis("2008-12-25 15:30:00.123"), 1230219000123); + + EXPECT_EQ(timestampToMillis("1970-01-01 00:00:00.127"), kMaxTinyint); + EXPECT_EQ(timestampToMillis("1969-12-31 23:59:59.872"), kMinTinyint); + EXPECT_EQ(timestampToMillis("1970-01-01 00:00:32.767"), kMaxSmallint); + EXPECT_EQ(timestampToMillis("1969-12-31 23:59:27.232"), kMinSmallint); + EXPECT_EQ(timestampToMillis("1970-01-25 20:31:23.647"), kMax); + EXPECT_EQ(timestampToMillis("1969-12-07 03:28:36.352"), kMin); + EXPECT_EQ(timestampToMillis("292278994-08-17T07:12:55.807"), kMaxLong); + EXPECT_EQ(timestampToMillis("-292275055-05-16T16:47:04.192"), kMinLong); +} + } // namespace } // namespace facebook::velox::functions::sparksql::test