Skip to content

Commit

Permalink
Add timestamp_micros, timestamp_millis, unix_micros, unix_millis Spar…
Browse files Browse the repository at this point in the history 
…k functions
  • Loading branch information
@zhli1142015
zhli1142015 committed Apr 11, 2024
1 parent efb0213 commit d4ea225
Show file tree
Hide file tree
Showing 4 changed files with 204 additions and 0 deletions.
23 changes: 23 additions & 0 deletions velox/docs/functions/spark/datetime.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,20 @@ These functions support TIMESTAMP and DATE input types.

SELECT second('2009-07-30 12:58:59'); -- 59

.. spark:function:: timestamp_micros(x) -> timestamp
Returns timestamp from the number of microseconds since UTC epoch.
Supported types are: TINYINT, SMALLINT, INTEGER and BIGINT.::

SELECT timestamp_micros(1230219000123123); -- '2008-12-25 15:30:00.123123'

.. spark:function:: timestamp_millis(x) -> timestamp
Returns timestamp from the number of milliseconds since UTC epoch.
Supported types are: TINYINT, SMALLINT, INTEGER and BIGINT.::

SELECT timestamp_millis(1230219000123); -- '2008-12-25 15:30:00.123'

.. spark:function:: to_unix_timestamp(string) -> integer
Alias for ``unix_timestamp(string) -> integer``.
Expand All @@ -244,6 +258,15 @@ These functions support TIMESTAMP and DATE input types.
SELECT unix_date('1970-01-02'); -- '1'
SELECT unix_date('1969-12-31'); -- '-1'

.. spark:function:: unix_micros(timestamp) -> bigint
Returns the number of microseconds since 1970-01-01 00:00:00 UTC.

.. spark:function:: unix_millis(timestamp) -> bigint
Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates
higher levels of precision.

.. spark:function:: unix_timestamp() -> integer
Returns the current UNIX timestamp in seconds.
Expand Down
47 changes: 47 additions & 0 deletions velox/functions/sparksql/DateTimeFunctions.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -731,4 +731,51 @@ struct MakeYMIntervalFunction {
result = totalMonths;
}
};

template <typename T>
struct TimestampToMicrosFunction {
VELOX_DEFINE_FUNCTION_TYPES(T);

FOLLY_ALWAYS_INLINE void call(
int64_t& result,
const arg_type<Timestamp>& timestamp) {
result = timestamp.toMicros();
}
};

template <typename T>
struct MicrosToTimestampFunction {
VELOX_DEFINE_FUNCTION_TYPES(T);

template <typename TInput>
FOLLY_ALWAYS_INLINE void call(
out_type<Timestamp>& result,
const TInput micros) {
result = Timestamp::fromMicros(micros);
}
};

template <typename T>
struct TimestampToMillisFunction {
VELOX_DEFINE_FUNCTION_TYPES(T);

FOLLY_ALWAYS_INLINE void call(
int64_t& result,
const arg_type<Timestamp>& timestamp) {
result = timestamp.toMillis();
}
};

template <typename T>
struct MillisToTimestampFunction {
VELOX_DEFINE_FUNCTION_TYPES(T);

template <typename TInput>
FOLLY_ALWAYS_INLINE void call(
out_type<Timestamp>& result,
const TInput millis) {
result = Timestamp::fromMillis(millis);
}
};

} // namespace facebook::velox::functions::sparksql
21 changes: 21 additions & 0 deletions velox/functions/sparksql/Register.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,27 @@ void registerFunctions(const std::string& prefix) {

VELOX_REGISTER_VECTOR_FUNCTION(udf_make_timestamp, prefix + "make_timestamp");

registerFunction<TimestampToMicrosFunction, int64_t, Timestamp>(
{prefix + "unix_micros"});
registerFunction<MicrosToTimestampFunction, Timestamp, int8_t>(
{prefix + "timestamp_micros"});
registerFunction<MicrosToTimestampFunction, Timestamp, int16_t>(
{prefix + "timestamp_micros"});
registerFunction<MicrosToTimestampFunction, Timestamp, int32_t>(
{prefix + "timestamp_micros"});
registerFunction<MicrosToTimestampFunction, Timestamp, int64_t>(
{prefix + "timestamp_micros"});
registerFunction<TimestampToMillisFunction, int64_t, Timestamp>(
{prefix + "unix_millis"});
registerFunction<MillisToTimestampFunction, Timestamp, int8_t>(
{prefix + "timestamp_millis"});
registerFunction<MillisToTimestampFunction, Timestamp, int16_t>(
{prefix + "timestamp_millis"});
registerFunction<MillisToTimestampFunction, Timestamp, int32_t>(
{prefix + "timestamp_millis"});
registerFunction<MillisToTimestampFunction, Timestamp, int64_t>(
{prefix + "timestamp_millis"});

// Register bloom filter function
registerFunction<BloomFilterMightContainFunction, bool, Varbinary, int64_t>(
{prefix + "might_contain"});
Expand Down
113 changes: 113 additions & 0 deletions velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ class DateTimeFunctionsTest : public SparkFunctionBaseTest {
static constexpr int16_t kMaxSmallint = std::numeric_limits<int16_t>::max();
static constexpr int8_t kMinTinyint = std::numeric_limits<int8_t>::min();
static constexpr int8_t kMaxTinyint = std::numeric_limits<int8_t>::max();
static constexpr int64_t kMinLong = std::numeric_limits<int64_t>::min();
static constexpr int64_t kMaxLong = std::numeric_limits<int64_t>::max();

protected:
void setQueryTimeZone(const std::string& timeZone) {
Expand Down Expand Up @@ -968,5 +970,116 @@ TEST_F(DateTimeFunctionsTest, makeYMInterval) {
fromYear(-178956971), "Integer overflow in make_ym_interval(-178956971)");
}

TEST_F(DateTimeFunctionsTest, microsToTimestamp) {
const auto microsToTimestamp = [&](int64_t micros) {
return evaluateOnce<Timestamp, int64_t>("timestamp_micros(c0)", micros);
};
EXPECT_EQ(
microsToTimestamp(1000000),
util::fromTimestampString("1970-01-01 00:00:01"));
EXPECT_EQ(
microsToTimestamp(1230219000123123),
util::fromTimestampString("2008-12-25 15:30:00.123123"));

EXPECT_EQ(
microsToTimestamp(kMaxTinyint),
util::fromTimestampString("1970-01-01 00:00:00.000127"));
EXPECT_EQ(
microsToTimestamp(kMinTinyint),
util::fromTimestampString("1969-12-31 23:59:59.999872"));
EXPECT_EQ(
microsToTimestamp(kMaxSmallint),
util::fromTimestampString("1970-01-01 00:00:00.032767"));
EXPECT_EQ(
microsToTimestamp(kMinSmallint),
util::fromTimestampString("1969-12-31 23:59:59.967232"));
EXPECT_EQ(
microsToTimestamp(kMax),
util::fromTimestampString("1970-01-01 00:35:47.483647"));
EXPECT_EQ(
microsToTimestamp(kMin),
util::fromTimestampString("1969-12-31 23:24:12.516352"));
EXPECT_EQ(
microsToTimestamp(kMaxLong),
util::fromTimestampString("294247-01-10 04:00:54.775807"));
EXPECT_EQ(
microsToTimestamp(kMinLong),
util::fromTimestampString("-290308-12-21 19:59:05.224192"));
}

TEST_F(DateTimeFunctionsTest, millisToTimestamp) {
const auto millisToTimestamp = [&](int64_t millis) {
return evaluateOnce<Timestamp, int64_t>("timestamp_millis(c0)", millis);
};
EXPECT_EQ(
millisToTimestamp(1000),
util::fromTimestampString("1970-01-01 00:00:01"));
EXPECT_EQ(
millisToTimestamp(1230219000123),
util::fromTimestampString("2008-12-25 15:30:00.123"));

EXPECT_EQ(
millisToTimestamp(kMaxTinyint),
util::fromTimestampString("1970-01-01 00:00:00.127"));
EXPECT_EQ(
millisToTimestamp(kMinTinyint),
util::fromTimestampString("1969-12-31 23:59:59.872"));
EXPECT_EQ(
millisToTimestamp(kMaxSmallint),
util::fromTimestampString("1970-01-01 00:00:32.767"));
EXPECT_EQ(
millisToTimestamp(kMinSmallint),
util::fromTimestampString("1969-12-31 23:59:27.232"));
EXPECT_EQ(
millisToTimestamp(kMax),
util::fromTimestampString("1970-01-25 20:31:23.647"));
EXPECT_EQ(
millisToTimestamp(kMin),
util::fromTimestampString("1969-12-07 03:28:36.352"));
EXPECT_EQ(
millisToTimestamp(kMaxLong),
util::fromTimestampString("292278994-08-17T07:12:55.807"));
EXPECT_EQ(
millisToTimestamp(kMinLong),
util::fromTimestampString("-292275055-05-16T16:47:04.192"));
}

TEST_F(DateTimeFunctionsTest, timestampToMicros) {
const auto timestampToMicros = [&](const StringView time) {
return evaluateOnce<int64_t, Timestamp>(
"unix_micros(c0)", util::fromTimestampString(time));
};
EXPECT_EQ(timestampToMicros("1970-01-01 00:00:01"), 1000000);
EXPECT_EQ(timestampToMicros("2008-12-25 15:30:00.123123"), 1230219000123123);

EXPECT_EQ(timestampToMicros("1970-01-01 00:00:00.000127"), kMaxTinyint);
EXPECT_EQ(timestampToMicros("1969-12-31 23:59:59.999872"), kMinTinyint);
EXPECT_EQ(timestampToMicros("1970-01-01 00:00:00.032767"), kMaxSmallint);
EXPECT_EQ(timestampToMicros("1969-12-31 23:59:59.967232"), kMinSmallint);
EXPECT_EQ(timestampToMicros("1970-01-01 00:35:47.483647"), kMax);
EXPECT_EQ(timestampToMicros("1969-12-31 23:24:12.516352"), kMin);
EXPECT_EQ(timestampToMicros("294247-01-10 04:00:54.775807"), kMaxLong);
EXPECT_EQ(
timestampToMicros("-290308-12-21 19:59:06.224192"), kMinLong + 1000000);
}

TEST_F(DateTimeFunctionsTest, timestampToMillis) {
const auto timestampToMillis = [&](const StringView time) {
return evaluateOnce<int64_t, Timestamp>(
"unix_millis(c0)", util::fromTimestampString(time));
};
EXPECT_EQ(timestampToMillis("1970-01-01 00:00:01"), 1000);
EXPECT_EQ(timestampToMillis("2008-12-25 15:30:00.123"), 1230219000123);

EXPECT_EQ(timestampToMillis("1970-01-01 00:00:00.127"), kMaxTinyint);
EXPECT_EQ(timestampToMillis("1969-12-31 23:59:59.872"), kMinTinyint);
EXPECT_EQ(timestampToMillis("1970-01-01 00:00:32.767"), kMaxSmallint);
EXPECT_EQ(timestampToMillis("1969-12-31 23:59:27.232"), kMinSmallint);
EXPECT_EQ(timestampToMillis("1970-01-25 20:31:23.647"), kMax);
EXPECT_EQ(timestampToMillis("1969-12-07 03:28:36.352"), kMin);
EXPECT_EQ(timestampToMillis("292278994-08-17T07:12:55.807"), kMaxLong);
EXPECT_EQ(timestampToMillis("-292275055-05-16T16:47:04.192"), kMinLong);
}

} // namespace
} // namespace facebook::velox::functions::sparksql::test

0 comments on commit d4ea225

Please sign in to comment.