Repository: vaexio/vaex

Issue:
[BUG-REPORT] blake3 0.3.0 breaking changes
Pinning blake3 to 0.2.1 works as a fix for the error below.

**Description**

Vaex-core's dataset hash array functionality was broken by a recent update in blake3-py

```
tests/foo.py:96: in test_dataframe
    vaex_df = vaex.from_dict(matrix)
.venv/lib/python3.9/site-packages/vaex/__init__.py:310: in from_dict
    return vaex.from_arrays(**data)
.venv/lib/python3.9/site-packages/vaex/__init__.py:365: in from_arrays
    dataset = vaex.dataset.DatasetArrays(arrays)
.venv/lib/python3.9/site-packages/vaex/dataset.py:1287: in __init__
    self._ids = frozendict({key: hash_array(array) for key, array in self._columns.items()})
.venv/lib/python3.9/site-packages/vaex/dataset.py:1287: in <dictcomp>
    self._ids = frozendict({key: hash_array(array) for key, array in self._columns.items()})
.venv/lib/python3.9/site-packages/vaex/dataset.py:165: in hash_array
    hash_info = hash_array_data(ar)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

ar = array([   0,    1,    2, ..., 9997, 9998, 9999])

    def hash_array_data(ar):
        # this function should stay consistent with all future versions
        # since this is the expensive part of the hashing
        if isinstance(ar, np.ndarray):
            ar = ar.ravel()
            if ar.dtype == np.object_:
                return {"type": "numpy", "data": str(uuid.uuid4()), "mask": None}
            if np.ma.isMaskedArray(ar):
                data_byte_ar = _to_bytes(ar.data)
                blake = blake3.blake3(data_byte_ar, multithreading=True)
                hash_data = {"type": "numpy", "data": blake.hexdigest(), "mask": None}
                if ar.mask is not True and ar.mask is not False and ar.mask is not np.True_ and ar.mask is not np.False_:
                    mask_byte_ar = _to_bytes(ar.mask)
                    blake = blake3.blake3(mask_byte_ar, multithreading=True)
                    hash_data["mask"] = blake.hexdigest()
                return hash_data
            else:
                try:
                    byte_ar = _to_bytes(ar)
                except ValueError:
                    byte_ar = ar.copy().view(np.uint8)
>               blake = blake3.blake3(byte_ar, multithreading=True)
E               TypeError: blake3.__new__() got an unexpected keyword argument 'multithreading'
```

**Software information**
 - Vaex version (`import vaex; vaex.__version__)`:
    ```
    >>> import vaex; vaex.__version__
    {'vaex-core': '4.7.0', 'vaex-hdf5': '0.11.1'}
    ```
 - Vaex was installed via: pip 
 - OS: macos 12.0.1 (21A559)

**Additional information**
Please state any supplementary information or provide additional context for the problem (e.g. screenshots, data, etc..).



Discussion:

Resolving patch:
diff --git a/packages/vaex-core/vaex/dataset.py b/packages/vaex-core/vaex/dataset.py
--- a/packages/vaex-core/vaex/dataset.py
+++ b/packages/vaex-core/vaex/dataset.py
@@ -23,8 +23,6 @@
 from vaex import encoding
 
 
-blake3 = vaex.utils.optional_import('blake3')
-
 logger = logging.getLogger('vaex.dataset')
 
 opener_classes = []
@@ -96,17 +94,17 @@ def _to_bytes(ar):
         return ar.copy().view(np.uint8)
 
 def hash_combine(*hashes):
-    blake = blake3.blake3(multithreading=False)
+    hasher = vaex.utils.create_hasher(large_data=False)
     for hash in hashes:
-        blake.update(hash.encode())
-    return blake.hexdigest()
+        hasher.update(hash.encode())
+    return hasher.hexdigest()
 
 
 def hash_slice(hash, start, end):
-    blake = blake3.blake3(hash.encode(), multithreading=False)
+    hasher = vaex.utils.create_hasher(hash.encode(), large_data=False)
     slice = np.array([start, end], dtype=np.int64)
-    blake.update(_to_bytes(slice))
-    return blake.hexdigest()
+    hasher.update(_to_bytes(slice))
+    return hasher.hexdigest()
 
 
 def hash_array_data(ar):
@@ -118,22 +116,22 @@ def hash_array_data(ar):
             return {"type": "numpy", "data": str(uuid.uuid4()), "mask": None}
         if np.ma.isMaskedArray(ar):
             data_byte_ar = _to_bytes(ar.data)
-            blake = blake3.blake3(data_byte_ar, multithreading=True)
-            hash_data = {"type": "numpy", "data": blake.hexdigest(), "mask": None}
+            hasher = vaex.utils.create_hasher(data_byte_ar, large_data=True)
+            hash_data = {"type": "numpy", "data": hasher.hexdigest(), "mask": None}
             if ar.mask is not True and ar.mask is not False and ar.mask is not np.True_ and ar.mask is not np.False_:
                 mask_byte_ar = _to_bytes(ar.mask)
-                blake = blake3.blake3(mask_byte_ar, multithreading=True)
-                hash_data["mask"] = blake.hexdigest()
+                hasher = vaex.utils.create_hasher(mask_byte_ar, large_data=True)
+                hash_data["mask"] = hasher.hexdigest()
             return hash_data
         else:
             try:
                 byte_ar = _to_bytes(ar)
             except ValueError:
                 byte_ar = ar.copy().view(np.uint8)
-            blake = blake3.blake3(byte_ar, multithreading=True)
-            hash_data = {"type": "numpy", "data": blake.hexdigest(), "mask": None}
+            hasher = vaex.utils.create_hasher(byte_ar, large_data=True)
+            hash_data = {"type": "numpy", "data": hasher.hexdigest(), "mask": None}
     elif isinstance(ar, (pa.Array, pa.ChunkedArray)):
-        blake = blake3.blake3(multithreading=True)
+        hasher = vaex.utils.create_hasher(large_data=True)
         buffer_hashes = []
         hash_data = {"type": "arrow", "buffers": buffer_hashes}
         if isinstance(ar, pa.ChunkedArray):
@@ -143,12 +141,8 @@ def hash_array_data(ar):
         for chunk in chunks:
             for buffer in chunk.buffers():
                 if buffer is not None:
-                    # TODO: we need to make a copy here, a memoryview would be better
-                    # or possible patch the blake module to accept a memoryview https://github.com/oconnor663/blake3-py/issues/9
-                    # or feed in the buffer in batches
-                    # blake.update(buffer)
-                    blake.update(memoryview(buffer))
-                    buffer_hashes.append(blake.hexdigest())
+                    hasher.update(memoryview(buffer))
+                    buffer_hashes.append(hasher.hexdigest())
                 else:
                     buffer_hashes.append(None)
     elif isinstance(ar, vaex.column.Column):
@@ -187,10 +181,10 @@ def hash_array(ar, hash_info=None, return_info=False):
             hash_info = hash_array_data(ar)
         keys = [HASH_VERSION]
         keys.append(hash_info['fingerprint'])
-    blake = blake3.blake3(multithreading=False)  # small amounts of data
+    hasher = vaex.utils.create_hasher(large_data=False)  # small amounts of data
     for key in keys:
-        blake.update(key.encode('ascii'))
-    hash = blake.hexdigest()
+        hasher.update(key.encode('ascii'))
+    hash = hasher.hexdigest()
     if return_info:
         hash_info['hash'] = hash
         hash_info[HASH_VERSION_KEY] = HASH_VERSION
diff --git a/packages/vaex-core/vaex/encoding.py b/packages/vaex-core/vaex/encoding.py
--- a/packages/vaex-core/vaex/encoding.py
+++ b/packages/vaex-core/vaex/encoding.py
@@ -14,8 +14,6 @@
 import vaex
 from .datatype import DataType
 
-import blake3
-
 
 registry = {}
 
@@ -401,8 +399,8 @@ def decode_dict(self, typename, values, **kwargs):
 
     def add_blob(self, buffer):
         bytes = memoryview(buffer).tobytes()
-        blake = blake3.blake3(bytes, multithreading=True)
-        blob_id = blake.hexdigest()
+        hasher = vaex.utils.create_hasher(bytes, large_data=True)
+        blob_id = hasher.hexdigest()
         self.blobs[blob_id] = bytes
         return f'blob:{blob_id}'
 
diff --git a/packages/vaex-core/vaex/utils.py b/packages/vaex-core/vaex/utils.py
--- a/packages/vaex-core/vaex/utils.py
+++ b/packages/vaex-core/vaex/utils.py
@@ -20,6 +20,7 @@
 from filelock import FileLock
 import pkg_resources
 
+import blake3
 import dask.utils
 import numpy as np
 import pyarrow as pa
@@ -968,3 +969,12 @@ def file_lock(name):
     os.makedirs(os.path.dirname(path), exist_ok=True)
     with FileLock(path):
         yield
+
+
+def create_hasher(initial_data=None, large_data=True):
+    """Creates a blake3 hasher"""
+    version = tuple(map(int, blake3.__version__.split(".")[:2]))
+    if version < (0, 3):
+        return blake3.blake3(initial_data, multithreading=large_data)
+    else:
+        return blake3.blake3(initial_data, max_threads=blake3.blake3.AUTO if large_data else 1)


Test patch: