Repository: dask/distributed

Issue:
"Cycle detected in Dask" with distributed scheduler
Some simple dataframe operations that were working previously broke on upgrading to 2.29; I `bisect`ed and traced it back to #4127. Certainly sounds related based on the traceback but I'm afraid I don't understand the change well enough to attempt a PR quite yet. It's especially curious to me that this low-level graph optimization logic functions differently when running with or without the distributed scheduler; if someone has a 1-sentence description of how that could be I'd be keen to hear it 🙂 

```
# dask 2.29, pandas 1.0.4
import dask.dataframe as dd
import distributed
import pandas as pd

if __name__ == "__main__":
    client = distributed.Client()  # commenting this line out fixes the issue
    ddf = dd.from_pandas(
        pd.DataFrame(
            dict(
                uid=range(50),
                enter_time=pd.date_range(start="2020-01-01", end="2020-09-01", periods=50, tz="UTC"),
            )
        ),
        npartitions=5,
    )
    ddf = ddf[ddf.uid.isin(range(29))].persist()
    ddf["local_time"] = ddf.enter_time.dt.tz_convert("US/Central")
    ddf["day"] = ddf.enter_time.dt.day_name()
    ddf = dd.categorical.categorize(ddf, columns=["day"], index=False)  # fails here, even w/o persisting or computing
```
Error:
```

  File "/Users/brett/Dropbox/Documents/distributed/distributed/client.py", line 1852, in _gather
    raise exception.with_traceback(traceback)
  File "/Users/brett/model/.venv/lib/python3.8/site-packages/dask/optimization.py", line 961, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/Users/brett/model/.venv/lib/python3.8/site-packages/dask/core.py", line 149, in get
    for key in toposort(dsk):
  File "/Users/brett/model/.venv/lib/python3.8/site-packages/dask/core.py", line 403, in toposort
    return _toposort(dsk, dependencies=dependencies)
  File "/Users/brett/model/.venv/lib/python3.8/site-packages/dask/core.py", line 384, in _toposort
    raise RuntimeError("Cycle detected in Dask: %s" % cycle)
RuntimeError: Cycle detected in Dask: ('getitem-d192ddb03fa1cbad438e5f0acd7b89ce', 4)->('getitem-d192ddb03fa1cbad438e5f0acd7b89ce', 4)
```

cc @madsbk @mrocklin from that PR


Discussion:
Thank you for the excellent bug report @bnaul 

I have an async version (sort of) of this here (although the sync compute in the categorize call makes this harder to make clean):

```python
import dask.dataframe as dd
import distributed
import pandas as pd
from distributed.utils_test import gen_cluster

@gen_cluster(client=True)
async def test_foo(c, s, a, b):
    ddf = dd.from_pandas(
        pd.DataFrame(
            dict(
                uid=range(50),
                enter_time=pd.date_range(start="2020-01-01", end="2020-09-01", periods=50, tz="UTC"),
            )
        ),
        npartitions=5,
    )

    ddf = ddf[ddf.uid.isin(range(29))].persist()
    ddf["local_time"] = ddf.enter_time.dt.tz_convert("US/Central")
    ddf["day"] = ddf.enter_time.dt.day_name()
    ddf = await c.submit(dd.categorical.categorize, ddf, columns=["day"], index=False)  # fails here, even w/o persisting or computing
```

Placing a `breakpoint()` in dask/core.py where the error occurs we find that we're within a worker thread calling the local dask.get function on a subgraph callable with the following state:

```python
(Pdb)   /home/mrocklin/miniconda/lib/python3.8/threading.py(890)_bootstrap()
-> self._bootstrap_inner()
  /home/mrocklin/miniconda/lib/python3.8/threading.py(932)_bootstrap_inner()
-> self.run()
  /home/mrocklin/miniconda/lib/python3.8/threading.py(870)run()
-> self._target(*self._args, **self._kwargs)
  /home/mrocklin/workspace/distributed/distributed/threadpoolexecutor.py(55)_worker()
-> task.run()
(Pdb)   /home/mrocklin/workspace/distributed/distributed/_concurrent_futures_thread.py(65)run()
-> result = self.fn(*self.args, **self.kwargs)
  /home/mrocklin/workspace/distributed/distributed/worker.py(3411)apply_function()
-> result = function(*args, **kwargs)
  /home/mrocklin/workspace/dask/dask/optimization.py(961)__call__()
-> return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  /home/mrocklin/workspace/dask/dask/core.py(149)get()
-> for key in toposort(dsk):
  /home/mrocklin/workspace/dask/dask/core.py(404)toposort()
-> return _toposort(dsk, dependencies=dependencies)
> /home/mrocklin/workspace/dask/dask/core.py(385)_toposort()
-> raise RuntimeError("Cycle detected in Dask: %s" % cycle)
(Pdb) {('assign-24dac40652e1c815a06eaa61fbad20fc', 0): (subgraph_callable,
                                                  ('getitem-f30f12591e4e17656c774c743c51c834',
                                                   0),
                                                  'local_time',
                                                  (subgraph_callable,
                                                   (subgraph_callable,
                                                    ('getitem-f30f12591e4e17656c774c743c51c834',
                                                     0),
                                                    'enter_time'),
                                                   'dt',
                                                   'tz_convert',
                                                   ('US/Central',),
                                                   {},
                                                   'getitem-b00ec70e1635eb1519f6186285bac19e')),
 ('get-categories-chunk-88dc81c22982bbd4dc1124fda0f738e3', 0): (<function _get_categories at 0x7f8d7ce8f550>,
                                                                (subgraph_callable,
                                                                 ('assign-24dac40652e1c815a06eaa61fbad20fc',
                                                                  0),
                                                                 'day',
                                                                 (subgraph_callable,
                                                                  (subgraph_callable,
                                                                   ('assign-24dac40652e1c815a06eaa61fbad20fc',
                                                                    0),
                                                                   'enter_time'),
                                                                  'dt',
                                                                  'day_name',
                                                                  (),
                                                                  {},
                                                                  'getitem-0efd070d7ab213b1cc51ca32bd1697ac')),
                                                                ['day'],
                                                                False),
 ('getitem-f30f12591e4e17656c774c743c51c834', 0): ('getitem-f30f12591e4e17656c774c743c51c834',
                                                   0)}(Pdb) 
```

Note that the last key-value pair is a self-alias.  A key maps to itself.  

Placing a breakpoint in the `_graph_to_futures` method, and then continuing past the first two calls, reveals that we were pointing a key to a future within a subgraph-callable:

```
> /home/mrocklin/workspace/distributed/distributed/client.py(2558)_graph_to_futures()
-> for k, v in dsk.items():
(Pdb) pp dsk
{('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 0): (subgraph_callable,),
 ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 1): (subgraph_callable,),
 ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 2): (subgraph_callable,),
 ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 3): (subgraph_callable,),
 ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 4): (subgraph_callable,),
 ('get-categories-agg-33d568b674c4441a3aa62557da73a622', 0): (<function _get_categories_agg at 0x7feda06a25e0>,
                                                              [('get-categories-chunk-33d568b674c4441a3aa62557da73a622',
                                                                0),
                                                               ('get-categories-chunk-33d568b674c4441a3aa62557da73a622',
                                                                1),
                                                               ('get-categories-chunk-33d568b674c4441a3aa62557da73a622',
                                                                2),
                                                               ('get-categories-chunk-33d568b674c4441a3aa62557da73a622',
                                                                3),
                                                               ('get-categories-chunk-33d568b674c4441a3aa62557da73a622',
                                                                4)]),
 ('get-categories-chunk-33d568b674c4441a3aa62557da73a622', 0): ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d',
                                                                0),
 ('get-categories-chunk-33d568b674c4441a3aa62557da73a622', 1): ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d',
                                                                1),
 ('get-categories-chunk-33d568b674c4441a3aa62557da73a622', 2): ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d',
                                                                2),
 ('get-categories-chunk-33d568b674c4441a3aa62557da73a622', 3): ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d',
                                                                3),
 ('get-categories-chunk-33d568b674c4441a3aa62557da73a622', 4): ('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d',
                                                                4)}
(Pdb) pp dsk[('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 0)]
(subgraph_callable,)
(Pdb) pp dsk[('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 0)][0]
subgraph_callable
(Pdb) pp dsk[('assign-dt-get-categories-chunk-getitem-ab0cd7fd7bb1328f574f678fc91d734d', 0)][0].dsk
{('assign-7dcfa22a4a750f8e38f122f98789fbe0', 0): (subgraph_callable,
                                                  ('getitem-ab0cd7fd7bb1328f574f678fc91d734d',
                                                   0),
                                                  'local_time',
                                                  (subgraph_callable,
                                                   (subgraph_callable,
                                                    ('getitem-ab0cd7fd7bb1328f574f678fc91d734d',
                                                     0),
                                                    'enter_time'),
                                                   'dt',
                                                   'tz_convert',
                                                   ('US/Central',),
                                                   {},
                                                   'getitem-4d8d1ebf0c8bae8da5166d5e25edffcc')),
 ('get-categories-chunk-33d568b674c4441a3aa62557da73a622', 0): (<function _get_categories at 0x7feda06a2550>,
                                                                (subgraph_callable,
                                                                 ('assign-7dcfa22a4a750f8e38f122f98789fbe0',
                                                                  0),
                                                                 'day',
                                                                 (subgraph_callable,
                                                                  (subgraph_callable,
                                                                   ('assign-7dcfa22a4a750f8e38f122f98789fbe0',
                                                                    0),
                                                                   'enter_time'),
                                                                  'dt',
                                                                  'day_name',
                                                                  (),
                                                                  {},
                                                                  'getitem-b8c22458f992be50e35fddfbba8b66d3')),
                                                                ['day'],
                                                                False),
 ('getitem-ab0cd7fd7bb1328f574f678fc91d734d', 0): <Future: finished, type: pandas.DataFrame, key: ('getitem-ab0cd7fd7bb1328f574f678fc91d734d', 0)>}
```

This seems to get successfully unpacked into a normal key by the time we hit the worker (we see `(getitem, ...) -> (getitem, ...)`, but we don't seem to register the need for the data when running the task.

When I go back to 2.28.0 I find that the future has been unpacked, and stringified

```python
(Pdb) pp dsk3["('assign-dt-get-categories-chunk-getitem-567cecae99e5b7af9b4be6e38f90b00c', 0)"][0].dsk
{('assign-9038de3814b3841dc0285b09dad9ce77', 0): (subgraph_callable,
                                                  ('getitem-567cecae99e5b7af9b4be6e38f90b00c',
                                                   0),
                                                  'local_time',
                                                  (subgraph_callable,
                                                   (subgraph_callable,
                                                    ('getitem-567cecae99e5b7af9b4be6e38f90b00c',
                                                     0),
                                                    'enter_time'),
                                                   'dt',
                                                   'tz_convert',
                                                   ('US/Central',),
                                                   {},
                                                   'getitem-82336a9b4aafe4e7f2dfc402220cc92d')),
 ('get-categories-chunk-08e3450ed96e1369df21f8c33e51e374', 0): (<function _get_categories at 0x7f014b821550>,
                                                                (subgraph_callable,
                                                                 ('assign-9038de3814b3841dc0285b09dad9ce77',
                                                                  0),
                                                                 'day',
                                                                 (subgraph_callable,
                                                                  (subgraph_callable,
                                                                   ('assign-9038de3814b3841dc0285b09dad9ce77',
                                                                    0),
                                                                   'enter_time'),
                                                                  'dt',
                                                                  'day_name',
                                                                  (),
                                                                  {},
                                                                  'getitem-8c1c6c4ccca3d109ff78a6ecd2ddccf6')),
                                                                ['day'],
                                                                False),
 ('getitem-567cecae99e5b7af9b4be6e38f90b00c', 0): "('getitem-567cecae99e5b7af9b4be6e38f90b00c', "
                                                  '0)'}
```

(Please excuse the fact that the `getitem-` suffixes change.  They're random in this case and I got these notes from different runs.)

My guess, @madsbk , is that we need to apply some of the rountines like `unpack_remotedata` to the subgraphcallables of a Blockwise, and not just to the external keys.  Thoughts?
> My guess, @madsbk , is that we need to apply some of the rountines like `unpack_remotedata` to the subgraphcallables of a Blockwise, and not just to the external keys. Thoughts?

In `2.29.0` and `master` we do call `unpack_remotedata()` on subgraphcallable but we call `unpack_remotedata(..., byte_keys=False)` instead of `byte_keys=True` and then later we call `str_graph()` to _stringify_ everything. However, `str_graph()` doesn't _dive_ into `SubgraphCallable`, which I think is the problem.

I can work on a solution where `str_graph()` also handles `SubgraphCallable` later today or tomorrow.





Thanks Mads.  From my perspective fixing this regression is probably higher
priority than other HLG work.

On Mon, Oct 5, 2020 at 1:00 AM Mads R. B. Kristensen <
notifications@github.com> wrote:

> My guess, @madsbk <https://github.com/madsbk> , is that we need to apply
> some of the rountines like unpack_remotedata to the subgraphcallables of
> a Blockwise, and not just to the external keys. Thoughts?
>
> In 2.29.0 and master we do call unpack_remotedata() on subgraphcallable
> but we call unpack_remotedata(..., byte_keys=False) instead of
> byte_keys=True and then later we call str_graph() to *stringify*
> everything. However, str_graph() doesn't *dive* into SubgraphCallable,
> which I think is the problem.
>
> I can work on a solution where str_graph() also handles SubgraphCallable
> later today or tomorrow.
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/4145#issuecomment-703469519>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AACKZTG6ECVDN2WE6IS6LULSJF4JHANCNFSM4SDVM2AA>
> .
>


Resolving patch:
diff --git a/distributed/utils.py b/distributed/utils.py
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -36,6 +36,7 @@
 
 import dask
 from dask import istask
+from dask.optimization import SubgraphCallable
 
 # provide format_bytes here for backwards compatibility
 from dask.utils import (  # noqa
@@ -777,17 +778,31 @@ def _maybe_complex(task):
 
 
 def convert(task, dsk, extra_values):
-    if type(task) is list:
+    typ = type(task)
+    if typ is tuple and task:
+        if type(task[0]) is SubgraphCallable:
+            sc = task[0]
+            return (
+                SubgraphCallable(
+                    convert(sc.dsk, dsk, extra_values),
+                    sc.outkey,
+                    convert(sc.inkeys, dsk, extra_values),
+                    sc.name,
+                ),
+            ) + tuple(convert(x, dsk, extra_values) for x in task[1:])
+        elif callable(task[0]):
+            return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
+    if typ is list:
         return [convert(v, dsk, extra_values) for v in task]
-    if type(task) is dict:
+    if typ is dict:
         return {k: convert(v, dsk, extra_values) for k, v in task.items()}
-    if istask(task):
-        return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
     try:
         if task in dsk or task in extra_values:
             return tokey(task)
     except TypeError:
         pass
+    if typ is tuple:  # If the tuple itself isn't a key, check its elements
+        return tuple(convert(v, dsk, extra_values) for v in task)
     return task
 
 


Test patch:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6165,3 +6165,28 @@ async def test_mixed_compression(cleanup):
                     x = da.ones((10000, 10000))
                     y = x + x.T
                     await c.compute(y.sum())
+
+
+@gen_cluster(client=True)
+async def test_futures_in_subgraphs(c, s, a, b):
+    """Regression test of <https://github.com/dask/distributed/issues/4145>"""
+
+    dd = pytest.importorskip("dask.dataframe")
+    import pandas as pd
+
+    ddf = dd.from_pandas(
+        pd.DataFrame(
+            dict(
+                uid=range(50),
+                enter_time=pd.date_range(
+                    start="2020-01-01", end="2020-09-01", periods=50, tz="UTC"
+                ),
+            )
+        ),
+        npartitions=5,
+    )
+
+    ddf = ddf[ddf.uid.isin(range(29))].persist()
+    ddf["local_time"] = ddf.enter_time.dt.tz_convert("US/Central")
+    ddf["day"] = ddf.enter_time.dt.day_name()
+    ddf = await c.submit(dd.categorical.categorize, ddf, columns=["day"], index=False)
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -49,6 +49,7 @@
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
+from dask.optimization import SubgraphCallable
 
 
 def test_All(loop):
@@ -217,6 +218,11 @@ def test_str_graph():
         assert all(isinstance(k, str) for k in sdsk)
         assert dask.get(dsk, keys) == dask.get(sdsk, skeys)
 
+    dsk = {("y", 1): (SubgraphCallable({"x": ("y", 1)}, "x", (("y", 1),)), (("z", 1),))}
+    dsk = str_graph(dsk, extra_values=(("z", 1),))
+    assert dsk["('y', 1)"][0].dsk["x"] == "('y', 1)"
+    assert dsk["('y', 1)"][1][0] == "('z', 1)"
+
 
 def test_maybe_complex():
     assert not _maybe_complex(1)