[VL] Add spark monotonically_increasing_id function support (#4954)

apache · Mar 18, 2024 · 9bacf0f · 9bacf0f
1 parent 91d0275
commit 9bacf0f
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 1 deletion.
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
@@ -175,6 +175,7 @@ object CHExpressionUtil {
     ARRAY_EXCEPT -> DefaultValidator(),
     ARRAY_REPEAT -> DefaultValidator(),
     DATE_FROM_UNIX_DATE -> DefaultValidator(),
+    MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
     SKEWNESS -> DefaultValidator()
   )
 }
diff --git a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
@@ -444,6 +444,13 @@ class VeloxFunctionsValidateSuite extends VeloxWholeStageTransformerSuite {
     }
   }
 
+  test("Test monotonically_increasing_id function") {
+    runQueryAndCompare("""SELECT monotonically_increasing_id(), l_orderkey
+                         | from lineitem limit 1""".stripMargin) {
+      checkOperatorMatch[ProjectExecTransformer]
+    }
+  }
+
   test("Test hex function") {
     runQueryAndCompare("SELECT hex(l_partkey), hex(l_shipmode) FROM lineitem limit 1") {
       checkOperatorMatch[ProjectExecTransformer]

diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md
@@ -416,7 +416,7 @@ Gluten supports 199 functions. (Draw to right to see all data types)
 | java_method                   |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | least                         | least                  | least                 | S      |                        |         |      |       |     | S    | S     | S      | S    | S         |        |         |      |        |          |       |      |        |      |
 | md5                           | md5                    |                       | S      |                        |         | S    |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
-| monotonically_increasing_id   |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
+| monotonically_increasing_id   |                        |                       |   S     |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | nanvl                         |                        |                       | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | nvl                           |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | nvl2                          |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |

diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
@@ -242,6 +242,7 @@ object ExpressionMappings {
     Sig[CheckOverflow](CHECK_OVERFLOW),
     Sig[MakeDecimal](MAKE_DECIMAL),
     Sig[PromotePrecision](PROMOTE_PRECISION),
+    Sig[MonotonicallyIncreasingID](MONOTONICALLY_INCREASING_ID),
     // Decimal
     Sig[UnscaledValue](UNSCALED_VALUE)
   ) ++ SparkShimLoader.getSparkShims.expressionMappings

diff --git a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
@@ -262,6 +262,7 @@ object ExpressionNames {
   final val CHECK_OVERFLOW = "check_overflow"
   final val MAKE_DECIMAL = "make_decimal"
   final val PROMOTE_PRECISION = "promote_precision"
+  final val MONOTONICALLY_INCREASING_ID = "monotonically_increasing_id"
 
   // Directly use child expression transformer
   final val KNOWN_FLOATING_POINT_NORMALIZED = "known_floating_point_normalized"