[VL] Map date_format to a Velox function name (apache#3878)

zhanglistar · Nov 30, 2023 · b1bd3df · b1bd3df
1 parent 616029e
commit b1bd3df
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 14 deletions.
diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc
@@ -298,8 +298,8 @@ std::unordered_map<std::string, std::string> SubstraitParser::substraitVeloxFunc
     {"bit_and_merge", "bitwise_and_agg_merge"},
     {"collect_set", "array_distinct"},
     {"murmur3hash", "hash_with_seed"},
-    {"modulus", "mod"} /*Presto functions.*/
-};
+    {"modulus", "mod"}, /*Presto functions.*/
+    {"date_format", "format_datetime"}};
 
 const std::unordered_map<std::string, std::string> SubstraitParser::typeMap_ = {
     {"bool", "BOOLEAN"},

diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -44,7 +44,6 @@ static const std::unordered_set<std::string> kBlackList = {
     "json_array_length",
     "from_unixtime",
     "repeat",
-    "date_format",
     "trunc",
     "sequence",
     "posexplode",

diff --git a/...en-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxSQLQueryTestSettings.scala b/...en-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxSQLQueryTestSettings.scala
@@ -45,8 +45,9 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings {
       "current_database_catalog.sql",
       "date.sql",
       "datetime-formatting-invalid.sql",
-      "datetime-formatting-legacy.sql",
-      "datetime-formatting.sql",
+      // Velox had different handling for some illegal cases
+      // "datetime-formatting-legacy.sql",
+      // "datetime-formatting.sql",
       "datetime-legacy.sql",
       "datetime-parsing-invalid.sql",
       "datetime-parsing-legacy.sql",

diff --git a/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
@@ -288,8 +288,8 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
 
   // Modified based on vanilla spark to explicitly set timezone in config.
   test(GlutenTestConstants.GLUTEN_TEST + "DateFormat") {
-    val PST_OPT = Option(PST.getId)
-    val JST_OPT = Option(JST.getId)
+    val PST_OPT = Option("America/Los_Angeles")
+    val JST_OPT = Option("Asia/Tokyo")
 
     Seq("legacy", "corrected").foreach {
       legacyParserPolicy =>
@@ -331,7 +331,7 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
 
         withSQLConf(
           SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
-          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) {
           checkEvaluation(
             DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT),
             "2015")

diff --git a/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
@@ -288,8 +288,8 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
 
   // Modified based on vanilla spark to explicitly set timezone in config.
   test(GLUTEN_TEST + "DateFormat") {
-    val PST_OPT = Option(PST.getId)
-    val JST_OPT = Option(JST.getId)
+    val PST_OPT = Option("America/Los_Angeles")
+    val JST_OPT = Option("Asia/Tokyo")
 
     Seq("legacy", "corrected").foreach {
       legacyParserPolicy =>
@@ -331,7 +331,7 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
 
         withSQLConf(
           SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
-          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) {
           checkEvaluation(
             DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT),
             "2015")

diff --git a/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
@@ -288,8 +288,8 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
 
   // Modified based on vanilla spark to explicitly set timezone in config.
   test(GLUTEN_TEST + "DateFormat") {
-    val PST_OPT = Option(PST.getId)
-    val JST_OPT = Option(JST.getId)
+    val PST_OPT = Option("America/Los_Angeles")
+    val JST_OPT = Option("Asia/Tokyo")
 
     Seq("legacy", "corrected").foreach {
       legacyParserPolicy =>
@@ -331,7 +331,7 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr
 
         withSQLConf(
           SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy,
-          SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) {
+          SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) {
           checkEvaluation(
             DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT),
             "2015")