Repository: dask/dask

Issue:
Flag for raising an error when `normalize_object` doesn't find `__dask_tokenize__`
I ran into a bit of a nasty surprised when I realized some of my Delayed objects resulted in a different token when I called `tokenize()`. This is because in my data I had some objects which didn't have `__dask_tokenize__`. I fixed those errors, but I'm afraid they might come up again silently. It would be amazing if I could set a flag that would raise an error any time that I tried to tokenize something which contained anything that couldn't be tokenized deterministically. I'm thinking something in `dask.base.normalize_object` that raises an error instead of returning `uuid.uuid4().hex`.

Thanks!


Discussion:
Thanks for raising an issue @STOpandthink! I can imagine this was tedious to track down. Adding a config option to raise an error or warn when falling back to a random token (which is `False` by default) seems like a nice addition for certain use cases. Is this something you're interested in working on? (no obligation though)
Hi @jrbourbeau, could you please elaborate on the "config option" part?
I suppose, based on the wording, you're referring to something other than a flag that's passed to a particular function or set of functions.
Dask has it's own [configuration system](https://docs.dask.org/en/latest/configuration.html) that let's user customize some of Dask's behavior (e.g. logging verbosity). The proposal here is to add a new configuration option that controls if users are informed when [`dask.base.normalize_object`](https://github.com/dask/dask/blob/1767fedd2f0662ba2fd6a9caebf2d3595ca8046c/dask/base.py#L838) isn't able to return a deterministic result and falls back to returning a [random `uuid`](https://github.com/dask/dask/blob/1767fedd2f0662ba2fd6a9caebf2d3595ca8046c/dask/base.py#L842).

Resolving patch:
diff --git a/dask/base.py b/dask/base.py
--- a/dask/base.py
+++ b/dask/base.py
@@ -839,7 +839,14 @@ def func(seq):
         try:
             return list(map(normalize_token, seq))
         except RecursionError:
-            return str(uuid.uuid4())
+            if not config.get("tokenize.ensure-deterministic"):
+                return uuid.uuid4().hex
+
+            raise RuntimeError(
+                f"Sequence {str(seq)} cannot be deterministically hashed. Please, see "
+                "https://docs.dask.org/en/latest/custom-collections.html#implementing-deterministic-hashing "
+                "for more information"
+            )
 
     return type(seq).__name__, func(seq)
 
@@ -859,7 +866,18 @@ def normalize_object(o):
     method = getattr(o, "__dask_tokenize__", None)
     if method is not None:
         return method()
-    return normalize_function(o) if callable(o) else uuid.uuid4().hex
+
+    if callable(o):
+        return normalize_function(o)
+
+    if not config.get("tokenize.ensure-deterministic"):
+        return uuid.uuid4().hex
+
+    raise RuntimeError(
+        f"Object {str(o)} cannot be deterministically hashed. Please, see "
+        "https://docs.dask.org/en/latest/custom-collections.html#implementing-deterministic-hashing "
+        "for more information"
+    )
 
 
 function_cache = {}
@@ -1032,7 +1050,15 @@ def normalize_array(x):
                     data = hash_buffer_hex(pickle.dumps(x, pickle.HIGHEST_PROTOCOL))
                 except Exception:
                     # pickling not supported, use UUID4-based fallback
-                    data = uuid.uuid4().hex
+                    if not config.get("tokenize.ensure-deterministic"):
+                        data = uuid.uuid4().hex
+                    else:
+                        raise RuntimeError(
+                            f"``np.ndarray`` with object ``dtype`` {str(x)} cannot "
+                            "be deterministically hashed. Please, see "
+                            "https://docs.dask.org/en/latest/custom-collections.html#implementing-deterministic-hashing "  # noqa: E501
+                            "for more information"
+                        )
         else:
             try:
                 data = hash_buffer_hex(x.ravel(order="K").view("i1"))


Test patch:
diff --git a/dask/tests/test_base.py b/dask/tests/test_base.py
--- a/dask/tests/test_base.py
+++ b/dask/tests/test_base.py
@@ -128,9 +128,8 @@ def test_tokenize_numpy_scalar_string_rep():
 
 @pytest.mark.skipif("not np")
 def test_tokenize_numpy_array_on_object_dtype():
-    assert tokenize(np.array(["a", "aa", "aaa"], dtype=object)) == tokenize(
-        np.array(["a", "aa", "aaa"], dtype=object)
-    )
+    a = np.array(["a", "aa", "aaa"], dtype=object)
+    assert tokenize(a) == tokenize(a)
     assert tokenize(np.array(["a", None, "aaa"], dtype=object)) == tokenize(
         np.array(["a", None, "aaa"], dtype=object)
     )
@@ -138,6 +137,17 @@ def test_tokenize_numpy_array_on_object_dtype():
         np.array([(1, "a"), (1, None), (1, "aaa")], dtype=object)
     ) == tokenize(np.array([(1, "a"), (1, None), (1, "aaa")], dtype=object))
 
+    # Trigger non-deterministic hashing for object dtype
+    class NoPickle:
+        pass
+
+    x = np.array(["a", None, NoPickle], dtype=object)
+    assert tokenize(x) != tokenize(x)
+
+    with dask.config.set({"tokenize.ensure-deterministic": True}):
+        with pytest.raises(RuntimeError, match="cannot be deterministically hashed"):
+            tokenize(x)
+
 
 @pytest.mark.skipif("not np")
 def test_tokenize_numpy_memmap_offset(tmpdir):
@@ -226,6 +236,23 @@ def test_normalize_base():
         assert normalize_token(i) is i
 
 
+def test_tokenize_object():
+    o = object()
+    # Defaults to non-deterministic tokenization
+    assert normalize_token(o) != normalize_token(o)
+
+    with dask.config.set({"tokenize.ensure-deterministic": True}):
+        with pytest.raises(RuntimeError, match="cannot be deterministically hashed"):
+            normalize_token(o)
+
+
+def test_tokenize_callable():
+    def my_func(a, b, c=1):
+        return a + b + c
+
+    assert tokenize(my_func) == tokenize(my_func)  # Consistent token
+
+
 @pytest.mark.skipif("not pd")
 def test_tokenize_pandas():
     a = pd.DataFrame({"x": [1, 2, 3], "y": ["4", "asd", None]}, index=[1, 2, 3])
@@ -335,6 +362,10 @@ def __dask_tokenize__(self):
     assert tokenize(a) == tokenize(a)
     assert tokenize(a) != tokenize(b)
 
+    for ensure in [True, False]:
+        with dask.config.set({"tokenize.ensure-deterministic": ensure}):
+            assert tokenize(a) == tokenize(a)
+
     # dispatch takes precedence
     before = tokenize(a)
     normalize_token.register(Foo, lambda self: self.x + 1)
@@ -439,12 +470,16 @@ def test_tokenize_dense_sparse_array(cls_name):
     assert tokenize(a) != tokenize(b)
 
 
-def test_tokenize_object_with_recursion_error_returns_uuid():
+def test_tokenize_object_with_recursion_error():
     cycle = dict(a=None)
     cycle["a"] = cycle
 
     assert len(tokenize(cycle)) == 32
 
+    with dask.config.set({"tokenize.ensure-deterministic": True}):
+        with pytest.raises(RuntimeError, match="cannot be deterministically hashed"):
+            tokenize(cycle)
+
 
 def test_tokenize_datetime_date():
     # Same date