[VL] Fix wrong result caused by missing metadata in hash registration (…

…#5355)
apache · Apr 10, 2024 · 8420d75 · 8420d75
1 parent cf5c60a
commit 8420d75
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 10 deletions.
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
@@ -372,6 +372,20 @@ class TestOperator extends VeloxWholeStageTransformerSuite {
     }
   }
 
+  test("hash") {
+    withTempView("t") {
+      Seq[(Integer, String)]((1, "a"), (2, null), (null, "b"))
+        .toDF("a", "b")
+        .createOrReplaceTempView("t")
+      runQueryAndCompare("select hash(a, b) from t") {
+        checkGlutenOperatorMatch[ProjectExecTransformer]
+      }
+      runQueryAndCompare("select xxhash64(a, b) from t") {
+        checkGlutenOperatorMatch[ProjectExecTransformer]
+      }
+    }
+  }
+
   test("decimal abs") {
     runQueryAndCompare("""
                          |select abs(cast (l_quantity * (-1.0) as decimal(12, 2))),

diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc
@@ -36,16 +36,6 @@ using namespace facebook;
 namespace gluten {
 namespace {
 void registerFunctionOverwrite() {
-  velox::exec::registerStatefulVectorFunction(
-      "murmur3hash",
-      velox::functions::sparksql::hashWithSeedSignatures(),
-      velox::functions::sparksql::makeHashWithSeed);
-
-  velox::exec::registerStatefulVectorFunction(
-      "xxhash64",
-      velox::functions::sparksql::xxhash64WithSeedSignatures(),
-      velox::functions::sparksql::makeXxHash64WithSeed);
-
   velox::functions::registerUnaryNumeric<RoundFunction>({"round"});
   velox::registerFunction<RoundFunction, int8_t, int8_t, int32_t>({"round"});
   velox::registerFunction<RoundFunction, int16_t, int16_t, int32_t>({"round"});

diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc
@@ -399,6 +399,7 @@ std::unordered_map<std::string, std::string> SubstraitParser::substraitVeloxFunc
     {"bitwise_or", "spark_bitwise_or"},
     {"bitwise_xor", "spark_bitwise_xor"},
     {"murmur3hash", "hash_with_seed"},
+    {"xxhash64", "xxhash64_with_seed"},
     {"modulus", "remainder"},
     {"date_format", "format_datetime"},
     {"collect_set", "set_agg"}};