Merge pull request #2318 from mabel-dev/joocer/issue2317

✨ prefer cython hash

opteryx/__version__.py

@@ -1,4 +1,4 @@
-__build__ = 1038
+__build__ = 1040
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

opteryx/compiled/joins/filter_join.pyx

@@ -11,6 +11,8 @@ import numpy as np
 
 cimport numpy as cnp
 from libc.stdint cimport int64_t
+from cpython.object cimport PyObject_Hash
+
 from opteryx.third_party.abseil.containers cimport FlatHashSet
 
 cpdef FlatHashSet filter_join_set(relation, list join_columns, FlatHashSet seen_hashes):
@@ -31,14 +33,14 @@ cpdef FlatHashSet filter_join_set(relation, list join_columns, FlatHashSet seen_
     if num_columns == 1:
         col = values_array[0, :]
         for i in range(len(col)):
-            hash_value = <int64_t>hash(col[i])
+            hash_value = PyObject_Hash(col[i])
             seen_hashes.insert(hash_value)
     else:
         for i in range(values_array.shape[1]):
             # Combine the hashes of each value in the row
             hash_value = 0
             for value in values_array[:, i]:
-                hash_value = <int64_t>(hash_value * 31 + hash(value))
+                hash_value = <int64_t>(hash_value * 31 + PyObject_Hash(value))
             seen_hashes.insert(hash_value)
 
     return seen_hashes
@@ -55,7 +57,7 @@ cpdef anti_join(relation, list join_columns, FlatHashSet seen_hashes):
     if num_columns == 1:
         col = values_array[0, :]
         for i in range(len(col)):
-            hash_value = <int64_t>hash(col[i])
+            hash_value = PyObject_Hash(col[i])
             if not seen_hashes.contains(hash_value):
                 index_buffer[idx_count] = i
                 idx_count += 1
@@ -64,7 +66,7 @@ cpdef anti_join(relation, list join_columns, FlatHashSet seen_hashes):
             # Combine the hashes of each value in the row
             hash_value = 0
             for value in values_array[:, i]:
-                hash_value = <int64_t>(hash_value * 31 + hash(value))
+                hash_value = <int64_t>(hash_value * 31 + PyObject_Hash(value))
             if not seen_hashes.contains(hash_value):
                 index_buffer[idx_count] = i
                 idx_count += 1
@@ -87,7 +89,7 @@ cpdef semi_join(relation, list join_columns, FlatHashSet seen_hashes):
     if num_columns == 1:
         col = values_array[0, :]
         for i in range(len(col)):
-            hash_value = <int64_t>hash(col[i])
+            hash_value = PyObject_Hash(col[i])
             if seen_hashes.contains(hash_value):
                 index_buffer[idx_count] = i
                 idx_count += 1
@@ -96,7 +98,7 @@ cpdef semi_join(relation, list join_columns, FlatHashSet seen_hashes):
             # Combine the hashes of each value in the row
             hash_value = 0
             for value in values_array[:, i]:
-                hash_value = <int64_t>(hash_value * 31 + hash(value))
+                hash_value = <int64_t>(hash_value * 31 + PyObject_Hash(value))
             if seen_hashes.contains(hash_value):
                 index_buffer[idx_count] = i
                 idx_count += 1

opteryx/compiled/joins/inner_join.pyx

@@ -11,6 +11,7 @@ import numpy
 from libc.stdint cimport uint8_t, int64_t
 
 from opteryx.third_party.abseil.containers cimport FlatHashMap
+from cpython.object cimport PyObject_Hash
 
 cpdef FlatHashMap abs_hash_join_map(relation, list join_columns):
     """
@@ -66,14 +67,14 @@ cpdef FlatHashMap abs_hash_join_map(relation, list join_columns):
     if num_columns == 1:
         col = values_array[0, :]
         for i in range(len(col)):
-            hash_value = <int64_t>hash(col[i])
+            hash_value = PyObject_Hash(col[i])
             ht.insert(hash_value, non_null_indices[i])
     else:
         for i in range(values_array.shape[1]):
             # Combine the hashes of each value in the row
             hash_value = 0
             for value in values_array[:, i]:
-                hash_value = <int64_t>(hash_value * 31 + hash(value))
+                hash_value = <int64_t>(hash_value * 31 + PyObject_Hash(value))
             ht.insert(hash_value, non_null_indices[i])
 
     return ht

opteryx/compiled/structures/hash_table.pyx

@@ -11,6 +11,7 @@ from libcpp.unordered_map cimport unordered_map
 from libcpp.unordered_set cimport unordered_set
 from libcpp.vector cimport vector
 from libc.stdint cimport int64_t, uint8_t
+from cpython.object cimport PyObject_Hash
 
 cimport numpy as cnp
 
@@ -64,7 +65,7 @@ cpdef tuple list_distinct(cnp.ndarray values, cnp.int64_t[::1] indices, HashSet
 
     for i in range(n):
         v = values[i]
-        hash_value = <int64_t>hash(v)
+        hash_value = PyObject_Hash(v)
         if seen_hashes.insert(hash_value):
             new_values[j] = v
             new_indices[j] = indices[i]
@@ -127,14 +128,14 @@ cpdef HashTable hash_join_map(relation, list join_columns):
     if num_columns == 1:
         col = values_array[0, :]
         for i in range(len(col)):
-            hash_value = <int64_t>hash(col[i])
+            hash_value = PyObject_Hash(col[i])
             ht.insert(hash_value, non_null_indices[i])
     else:
         for i in range(values_array.shape[1]):
             # Combine the hashes of each value in the row
             hash_value = 0
             for value in values_array[:, i]:
-                hash_value = <int64_t>(hash_value * 31 + hash(value))
+                hash_value = <int64_t>(hash_value * 31 + PyObject_Hash(value))
             ht.insert(hash_value, non_null_indices[i])
 
     return ht

opteryx/connectors/iceberg_connector.py

@@ -16,7 +16,6 @@
 from typing import Union
 
 import pyarrow
-import pyiceberg.types
 from orso.schema import FlatColumn
 from orso.schema import RelationSchema