Introduce simple date time formatter

velox/functions/lib/DateTimeFormatter.cpp

@@ -1697,4 +1697,129 @@ std::shared_ptr<DateTimeFormatter> buildJodaDateTimeFormatter(
   return builder.setType(DateTimeFormatterType::JODA).build();
 }
 
+std::shared_ptr<DateTimeFormatter> buildSimpleDateTimeFormatter(
+    const std::string_view& format,
+    bool lenient) {
+  if (format.empty()) {
+    VELOX_USER_FAIL("Invalid pattern specification");
+  }
+
+  DateTimeFormatterBuilder builder(format.size());
+  const char* cur = format.data();
+  const char* end = cur + format.size();
+
+  while (cur < end) {
+    const char* startTokenPtr = cur;
+
+    // Literal case
+    if (*startTokenPtr == '\'') {
+      // Case 1: 2 consecutive single quote
+      if (cur + 1 < end && *(cur + 1) == '\'') {
+        builder.appendLiteral("'");
+        cur += 2;
+      } else {
+        // Case 2: find closing single quote
+        int64_t count = numLiteralChars(startTokenPtr + 1, end);
+        if (count == -1) {
+          VELOX_USER_FAIL("No closing single quote for literal");
+        } else {
+          for (int64_t i = 1; i <= count; i++) {
+            builder.appendLiteral(startTokenPtr + i, 1);
+            if (*(startTokenPtr + i) == '\'') {
+              i += 1;
+            }
+          }
+          cur += count + 2;
+        }
+      }
+    } else {
+      int count = 1;
+      ++cur;
+      while (cur < end && *startTokenPtr == *cur) {
+        ++count;
+        ++cur;
+      }
+      switch (*startTokenPtr) {
+        case 'G':
+          builder.appendEra();
+          break;
+        case 'C':
+          builder.appendCenturyOfEra(count);
+          break;
+        case 'Y':
+          builder.appendYearOfEra(count);
+          break;
+        case 'x':
+          builder.appendWeekYear(count);
+          break;
+        case 'w':
+          builder.appendWeekOfWeekYear(count);
+          break;
+        case 'e':
+          builder.appendDayOfWeek1Based(count);
+          break;
+        case 'E':
+          builder.appendDayOfWeekText(count);
+          break;
+        case 'y':
+          builder.appendYear(count);
+          break;
+        case 'D':
+          builder.appendDayOfYear(count);
+          break;
+        case 'M':
+          if (count <= 2) {
+            builder.appendMonthOfYear(count);
+          } else {
+            builder.appendMonthOfYearText(count);
+          }
+          break;
+        case 'd':
+          builder.appendDayOfMonth(count);
+          break;
+        case 'a':
+          builder.appendHalfDayOfDay();
+          break;
+        case 'K':
+          builder.appendHourOfHalfDay(count);
+          break;
+        case 'h':
+          builder.appendClockHourOfHalfDay(count);
+          break;
+        case 'H':
+          builder.appendHourOfDay(count);
+          break;
+        case 'k':
+          builder.appendClockHourOfDay(count);
+          break;
+        case 'm':
+          builder.appendMinuteOfHour(count);
+          break;
+        case 's':
+          builder.appendSecondOfMinute(count);
+          break;
+        case 'S':
+          builder.appendFractionOfSecond(count);
+          break;
+        case 'z':
+          builder.appendTimeZone(count);
+          break;
+        case 'Z':
+          builder.appendTimeZoneOffsetId(count);
+          break;
+        default:
+          if (isalpha(*startTokenPtr)) {
+            VELOX_UNSUPPORTED("Specifier {} is not supported.", *startTokenPtr);
+          } else {
+            builder.appendLiteral(startTokenPtr, cur - startTokenPtr);
+          }
+          break;
+      }
+    }
+  }
+  DateTimeFormatterType type = lenient ? DateTimeFormatterType::LENIENT_SIMPLE
+                                       : DateTimeFormatterType::STRICT_SIMPLE;
+  return builder.setType(type).build();
+}
+
 } // namespace facebook::velox::functions

velox/functions/lib/DateTimeFormatter.h

@@ -23,7 +23,15 @@
 
 namespace facebook::velox::functions {
 
-enum class DateTimeFormatterType { JODA, MYSQL, UNKNOWN };
+// LENIENT_SIMPLE and STRICT_SIMPLE are respectively aligned with
+// java.text.SimpleDateFormat in lenient and non-lenient modes.
+enum class DateTimeFormatterType {
+  JODA,
+  MYSQL,
+  LENIENT_SIMPLE,
+  STRICT_SIMPLE,
+  UNKNOWN
+};
 
 enum class DateTimeFormatSpecifier : uint8_t {
   // Era, e.g: "AD"
@@ -209,6 +217,10 @@ std::shared_ptr<DateTimeFormatter> buildMysqlDateTimeFormatter(
 std::shared_ptr<DateTimeFormatter> buildJodaDateTimeFormatter(
     const std::string_view& format);
 
+std::shared_ptr<DateTimeFormatter> buildSimpleDateTimeFormatter(
+    const std::string_view& format,
+    bool lenient);
+
 } // namespace facebook::velox::functions
 
 template <>

velox/functions/sparksql/CMakeLists.txt

@@ -15,6 +15,7 @@
 add_subdirectory(specialforms)
 velox_add_library(
   velox_functions_spark
+  flags.cpp
   ArrayGetFunction.cpp
   ArraySort.cpp
   Bitwise.cpp

velox/functions/sparksql/DateTimeFunctions.h

@@ -23,8 +23,21 @@
 #include "velox/type/TimestampConversion.h"
 #include "velox/type/tz/TimeZoneMap.h"
 
+DECLARE_bool(spark_sql_legacy_timeParserPolicy);
+
 namespace facebook::velox::functions::sparksql {
 
+std::shared_ptr<DateTimeFormatter> getDateTimeFormatter(
+    const std::string_view& format,
+    bool lenient) {
+  if (FLAGS_spark_sql_legacy_timeParserPolicy) {
+    return buildSimpleDateTimeFormatter(format, lenient);
+  } else {
+    return buildJodaDateTimeFormatter(
+        std::string_view(format.data(), format.size()));
+  }
+}
+
 template <typename T>
 struct YearFunction : public InitSessionTimezone<T> {
   VELOX_DEFINE_FUNCTION_TYPES(T);
@@ -156,7 +169,7 @@ struct UnixTimestampParseFunction {
       const std::vector<TypePtr>& /*inputTypes*/,
       const core::QueryConfig& config,
       const arg_type<Varchar>* /*input*/) {
-    format_ = buildJodaDateTimeFormatter(kDefaultFormat_);
+    format_ = getDateTimeFormatter(kDefaultFormat_, false);
     setTimezone(config);
   }
 
@@ -207,8 +220,8 @@ struct UnixTimestampParseWithFormatFunction
       const arg_type<Varchar>* format) {
     if (format != nullptr) {
       try {
-        this->format_ = buildJodaDateTimeFormatter(
-            std::string_view(format->data(), format->size()));
+        this->format_ = getDateTimeFormatter(
+            std::string_view(format->data(), format->size()), false);
       } catch (const VeloxUserError&) {
         invalidFormat_ = true;
       }
@@ -228,8 +241,8 @@ struct UnixTimestampParseWithFormatFunction
     // Format error returns null.
     try {
       if (!isConstFormat_) {
-        this->format_ = buildJodaDateTimeFormatter(
-            std::string_view(format.data(), format.size()));
+        this->format_ = getDateTimeFormatter(
+            std::string_view(format.data(), format.size()), false);
       }
     } catch (const VeloxUserError&) {
       return false;
@@ -284,8 +297,8 @@ struct FromUnixtimeFunction {
 
  private:
   FOLLY_ALWAYS_INLINE void setFormatter(const arg_type<Varchar>& format) {
-    formatter_ = buildJodaDateTimeFormatter(
-        std::string_view(format.data(), format.size()));
+    formatter_ = getDateTimeFormatter(
+        std::string_view(format.data(), format.size()), false);
     maxResultSize_ = formatter_->maxResultSize(sessionTimeZone_);
   }
 
@@ -371,7 +384,7 @@ struct GetTimestampFunction {
       sessionTimeZone_ = tz::locateZone(sessionTimezoneName);
     }
     if (format != nullptr) {
-      formatter_ = buildJodaDateTimeFormatter(std::string_view(*format));
+      formatter_ = getDateTimeFormatter(std::string_view(*format), false);
       isConstantTimeFormat_ = true;
     }
   }
@@ -381,7 +394,7 @@ struct GetTimestampFunction {
       const arg_type<Varchar>& input,
       const arg_type<Varchar>& format) {
     if (!isConstantTimeFormat_) {
-      formatter_ = buildJodaDateTimeFormatter(std::string_view(format));
+      formatter_ = getDateTimeFormatter(std::string_view(format), false);
     }
     auto dateTimeResult = formatter_->parse(std::string_view(input));
     // Null as result for parsing error.

velox/functions/sparksql/flags.cpp

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gflags/gflags.h>
+
+DEFINE_bool(
+    spark_sql_legacy_timeParserPolicy,
+    false,
+    "When true, SIMPLE_STRICT or SIMPLE_LENIENT date formatter is used for "
+    "formatting and parsing. This is aligned with Spark legacy date formatting "
+    "and parsing behavior.");