Migrate hashing operations to pylibcudf

dependencies.yaml

@@ -838,16 +838,13 @@ dependencies:
           - moto>=4.0.8
           - s3fs>=2022.3.0
           - python-xxhash
-      - output_types: pyproject
+      - output_types: [pyproject, requirements]
         packages:
           - msgpack
           - &tokenizers tokenizers==0.15.2
           - &transformers transformers==4.39.3
           - tzdata
           - xxhash
-      - output_types: requirements
-        packages:
-          - xxhash
     specific:
       - output_types: [conda, requirements]
         matrices:

python/pylibcudf/pylibcudf/__init__.pxd

@@ -14,6 +14,7 @@ from . cimport (
     filling,
     groupby,
     hashing,
+    interop,
     join,
     json,
     labeling,
@@ -64,6 +65,7 @@ __all__ = [
     "gpumemoryview",
     "groupby",
     "hashing",
+    "interop",
     "join",
     "json",
     "lists",

python/pylibcudf/pylibcudf/hashing.pyx

@@ -64,7 +64,7 @@ cpdef Table murmurhash3_x64_128(
     ----------
     input : Table
         The table of columns to hash
-    seed : uint32_t
+    seed : uint64_t
         Optional seed value to use for the hash function
 
     Returns
@@ -94,7 +94,7 @@ cpdef Column xxhash_64(
     ----------
     input : Table
         The table of columns to hash
-    seed : uint32_t
+    seed : uint64_t
         Optional seed value to use for the hash function
 
     Returns

python/pylibcudf/pylibcudf/tests/conftest.py

@@ -20,9 +20,9 @@
 
 def _type_to_str(typ):
     if isinstance(typ, pa.ListType):
-        return f"list-{_type_to_str(typ.value_type)}"
+        return f"list[{_type_to_str(typ.value_type)}]"
     elif isinstance(typ, pa.StructType):
-        return f"struct-{'-'.join([_type_to_str(typ.field(i).type) for i in range(typ.num_fields)])}"
+        return f"struct[{', '.join([_type_to_str(typ.field(i).type) for i in range(typ.num_fields)])}]"
     else:
         return str(typ)
 

python/pylibcudf/pylibcudf/tests/test_hashing.py

@@ -6,11 +6,12 @@
 import mmh3
 import numpy as np
 import pyarrow as pa
-import pylibcudf as plc
 import pytest
 import xxhash
 from utils import assert_column_eq, assert_table_eq
 
+import pylibcudf as plc
+
 SEED = 0
 METHODS = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512"]
 
@@ -28,10 +29,24 @@ def scalar_to_binary(x):
         raise NotImplementedError
 
 
+def hash_single_uint32(val, seed=0):
+    return mmh3.hash(np.uint32(val).tobytes(), seed=seed, signed=False)
+
+
+def hash_combine_32(lhs, rhs):
+    return lhs ^ (rhs + 0x9E3779B9 + (lhs << 6) + (lhs >> 2))
+
+
+def uint_hash_combine_32(lhs, rhs):
+    lhs = np.uint32(lhs)
+    rhs = np.uint32(rhs)
+    return hash_combine_32(lhs, rhs)
+
+
 def libcudf_mmh3_x86_32(binary):
     seed = plc.hashing.LIBCUDF_DEFAULT_HASH_SEED
     hashval = mmh3.hash(binary, seed)
-    return seed ^ (hashval + 0x9E3779B9 + (seed << 6) + (seed >> 2))
+    return hash_combine_32(seed, hashval)
 
 
 @pytest.fixture(params=[pa.int64(), pa.float64(), pa.string(), pa.bool_()])
@@ -84,9 +99,11 @@ def python_hash_value(x, method):
 
 
 @pytest.mark.parametrize(
-    "method", ["sha1", "sha224", "sha256", "sha384", "sha512"]
+    "method", ["sha1", "sha224", "sha256", "sha384", "sha512", "md5"]
 )
-def test_hash_column_sha(pa_scalar_input_column, plc_scalar_input_tbl, method):
+def test_hash_column_sha_md5(
+    pa_scalar_input_column, plc_scalar_input_tbl, method
+):
     plc_hasher = getattr(plc.hashing, method)
 
     def py_hasher(val):
@@ -100,18 +117,6 @@ def py_hasher(val):
     assert_column_eq(got, expect)
 
 
-def test_hash_column_md5(pa_scalar_input_column, plc_scalar_input_tbl):
-    def py_hasher(val):
-        return hashlib.md5(scalar_to_binary(val)).hexdigest()
-
-    expect = pa.array(
-        [py_hasher(val) for val in pa_scalar_input_column.to_pylist()],
-        type=pa.string(),
-    )
-    got = plc.hashing.md5(plc_scalar_input_tbl)
-    assert_column_eq(got, expect)
-
-
 def test_hash_column_xxhash64(pa_scalar_input_column, plc_scalar_input_tbl):
     def py_hasher(val):
         return xxhash.xxh64(
@@ -161,7 +166,87 @@ def py_hasher(val):
     assert_column_eq(got, expect)
 
 
-# def test_murmurhash_x86_32_list_struct TODO
+@pytest.mark.filterwarnings("ignore::RuntimeWarning")
+def test_murmurhash3_x86_32_list():
+    pa_tbl = pa.Table.from_pydict(
+        {
+            "list": pa.array(
+                [[1, 2, 3], [4, 5, 6], [7, 8, 9]], type=pa.list_(pa.uint32())
+            )
+        }
+    )
+    plc_tbl = plc.interop.from_arrow(pa_tbl)
+
+    def hash_list(list_):
+        hash_value = uint_hash_combine_32(0, hash_single_uint32(len(list_)))
+
+        for element in list_:
+            hash_value = uint_hash_combine_32(
+                hash_value,
+                hash_single_uint32(
+                    element, seed=plc.hashing.LIBCUDF_DEFAULT_HASH_SEED
+                ),
+            )
+
+        final = uint_hash_combine_32(
+            plc.hashing.LIBCUDF_DEFAULT_HASH_SEED, hash_value
+        )
+        return final
+
+    expect = pa.array(
+        [hash_list(val) for val in pa_tbl["list"].to_pylist()],
+        type=pa.uint32(),
+    )
+    got = plc.hashing.murmurhash3_x86_32(
+        plc_tbl, plc.hashing.LIBCUDF_DEFAULT_HASH_SEED
+    )
+    assert_column_eq(got, expect)
+
+
+@pytest.mark.filterwarnings("ignore::RuntimeWarning")
+def test_murmurhash3_x86_32_struct():
+    pa_tbl = pa.table(
+        {
+            "struct": pa.array(
+                [
+                    {"a": 1, "b": 2, "c": 3},
+                    {"a": 4, "b": 5, "c": 6},
+                    {"a": 7, "b": 8, "c": 9},
+                ],
+                type=pa.struct(
+                    [
+                        pa.field("a", pa.uint32()),
+                        pa.field("b", pa.uint32(), pa.field("c", pa.uint32())),
+                    ]
+                ),
+            )
+        }
+    )
+    plc_tbl = plc.interop.from_arrow(pa_tbl)
+
+    def hash_struct(s):
+        seed = plc.hashing.LIBCUDF_DEFAULT_HASH_SEED
+        keys = list(s.keys())
+
+        combined_hash = hash_single_uint32(s[keys[0]], seed=seed)
+        combined_hash = uint_hash_combine_32(0, combined_hash)
+        combined_hash = uint_hash_combine_32(seed, combined_hash)
+
+        for key in keys[1:]:
+            current_hash = hash_single_uint32(s[key], seed=seed)
+            combined_hash = uint_hash_combine_32(combined_hash, current_hash)
+
+        return combined_hash
+
+    got = plc.hashing.murmurhash3_x86_32(
+        plc_tbl, plc.hashing.LIBCUDF_DEFAULT_HASH_SEED
+    )
+
+    expect = pa.array(
+        [hash_struct(val) for val in pa_tbl["struct"].to_pylist()],
+        type=pa.uint32(),
+    )
+    assert_column_eq(got, expect)
 
 
 def test_murmurhash3_x64_128(pa_scalar_input_column, plc_scalar_input_tbl):