Repository: dask/dask
File: dask/tests/test_tokenize.py (Python)

Code before:
pd = import_or_none("pandas")


def check_tokenize(*args, **kwargs):
    with dask.config.set({"tokenize.ensure-deterministic": True}):
        before = tokenize(*args, **kwargs)

        # Test idempotency (the same object tokenizes to the same value)
        after = tokenize(*args, **kwargs)

        assert before == after

        # Test same-interpreter determinism (two identical objects tokenize to the
        # same value as long as you do it on the same interpreter) We are not
        # particularly interested in a class that's never been pickled vs. one
        # that's already been pickled already (cloudpickle can introduce artifacts
        # on the first round-trip). We do care however about classes that have both
        # been through a serialization roundtrip at least once (not necessarily the
        # same amount of times).
        args2, kwargs2 = cloudpickle.loads(cloudpickle.dumps((args, kwargs)))
        args3, kwargs3 = cloudpickle.loads(cloudpickle.dumps((args, kwargs)))
        args3, kwargs3 = cloudpickle.loads(cloudpickle.dumps((args3, kwargs3)))

        tok2 = tokenize(*args2, **kwargs2)
        tok3 = tokenize(*args3, **kwargs3)
        assert tok2 == tok3

        # Skip: different interpreter determinism

    return before


def test_check_tokenize():
    # idempotent and deterministic
    check_tokenize(123)
    # returns tokenized value
    assert tokenize(123) == check_tokenize(123)

    # idempotent, but not deterministic
    class A:
        def __init__(self):
            self.tok = random.random()

        def __reduce__(self):
            return A, ()

        def __dask_tokenize__(self):
            return self.tok

    a = A()
    with pytest.raises(AssertionError):

Review comment (suggestion):
I'm not sure if this is what you are referring to in the comment above but don't we want this to be equal as before?
```suggestion
        assert tok2 == tok3 == before
```

Code after:
    """Test that tokenize() and normalize_token() properly clean up context
    variables at all times
    """
    from dask.base import _ensure_deterministic, _seen

    with pytest.raises(LookupError):
        _ensure_deterministic.get()
    with pytest.raises(LookupError):
        _seen.get()
    yield
    with pytest.raises(LookupError):
        _ensure_deterministic.get()
    with pytest.raises(LookupError):
        _seen.get()


def check_tokenize(*args, **kwargs):
    with dask.config.set({"tokenize.ensure-deterministic": True}):
        before = tokenize(*args, **kwargs)

        # Test idempotency (the same object tokenizes to the same value)
        after = tokenize(*args, **kwargs)

        assert before == after

        # Test same-interpreter determinism (two identical objects tokenize to the
        # same value as long as you do it on the same interpreter) We are not
        # particularly interested in a class that's never been pickled vs. one
        # that's already been pickled already (cloudpickle can introduce artifacts
        # on the first round-trip). We do care however about classes that have both
        # been through a serialization roundtrip at least once (not necessarily the
        # same amount of times).
        args2, kwargs2 = cloudpickle.loads(cloudpickle.dumps((args, kwargs)))
        args3, kwargs3 = cloudpickle.loads(cloudpickle.dumps((args, kwargs)))
        args3, kwargs3 = cloudpickle.loads(cloudpickle.dumps((args3, kwargs3)))

        tok2 = tokenize(*args2, **kwargs2)
        tok3 = tokenize(*args3, **kwargs3)
        assert tok2 == tok3

        # Skip: different interpreter determinism

    return before


def test_check_tokenize():
    # idempotent and deterministic
    check_tokenize(123)
    # returns tokenized value
    assert tokenize(123) == check_tokenize(123)