Repository: dask/dask

Issue:
Recursion error in DataFrame.set_index
`git bisect` traced this regression back to #8517:

```
import dask, distributed
client = distributed.Client()
# Create an already-sorted index; reset first so we can actually call .set_index()
ddf = dask.datasets.timeseries(end="2000-07-01").reset_index().astype({"timestamp": str})
ddf = ddf.set_index("timestamp", sorted=True)
ddf.to_csv("/tmp/dask_test/")
```
pre-#8517, this finishes in ~30s on my laptop; post-#8517, it fails with
```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-4-29e339cba513> in <module>
      1 from uuid import uuid4
----> 2 ddf.to_csv(f"gs://model_bigquery_tmp/{uuid4().hex}/")

~/Dropbox/Documents/dask/dask/dataframe/core.py in to_csv(self, filename, **kwargs)
   1621         from .io import to_csv
   1622 
-> 1623         return to_csv(self, filename, **kwargs)
   1624 
   1625     def to_sql(

~/Dropbox/Documents/dask/dask/dataframe/io/csv.py in to_csv(df, filename, single_file, encoding, mode, name_function, compression, compute, scheduler, storage_options, header_first_partition_only, compute_kwargs, **kwargs)
    955         import dask
    956 
--> 957         return list(dask.compute(*values, **compute_kwargs))
    958     else:
    959         return values

~/Dropbox/Documents/dask/dask/base.py in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    573         postcomputes.append(x.__dask_postcompute__())
    574 
--> 575     results = schedule(dsk, keys, **kwargs)
    576     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    577 

~/Dropbox/Documents/distributed/distributed/client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
   3002                     should_rejoin = False
   3003             try:
-> 3004                 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
   3005             finally:
   3006                 for f in futures.values():

~/Dropbox/Documents/distributed/distributed/client.py in gather(self, futures, errors, direct, asynchronous)
   2176             else:
   2177                 local_worker = None
-> 2178             return self.sync(
   2179                 self._gather,
   2180                 futures,

~/Dropbox/Documents/distributed/distributed/utils.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    307             return future
    308         else:
--> 309             return sync(
    310                 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
    311             )

~/Dropbox/Documents/distributed/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    374     if error:
    375         typ, exc, tb = error
--> 376         raise exc.with_traceback(tb)
    377     else:
    378         return result

~/Dropbox/Documents/distributed/distributed/utils.py in f()
    347                 future = asyncio.wait_for(future, callback_timeout)
    348             future = asyncio.ensure_future(future)
--> 349             result = yield future
    350         except Exception:
    351             error = sys.exc_info()

~/model/.venv/lib/python3.9/site-packages/tornado/gen.py in run(self)
    760 
    761                     try:
--> 762                         value = future.result()
    763                     except Exception:
    764                         exc_info = sys.exc_info()

~/Dropbox/Documents/distributed/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
   2039                             exc = CancelledError(key)
   2040                         else:
-> 2041                             raise exception.with_traceback(traceback)
   2042                         raise exc
   2043                     if errors == "skip":

Exception: RecursionError('maximum recursion depth exceeded in __instancecheck__')
```
and also eats up an enormous amount of memory in the process (reaches several times higher than peak memory in the older version before giving up).

I haven't been able to reproduce the same error without the `distributed.Client()`, the computation seems to just hang forever with other schedulers...will keep playing around with the size of dataset/partitions and see if I can get something breaking w/o `distributed`.

EDIT: it did finally complete after ~10 minutes with `scheduler="threads"` (compared to ~90s on 2022.2.1), no recursion error
EDIT 2: couldn't reproduce the error w/ `processes` but I did manage to brick my OS by using so much memory 😅 

cc @jsignell @gjoseph92 
Recursion error in DataFrame.set_index
`git bisect` traced this regression back to #8517:

```
import dask, distributed
client = distributed.Client()
# Create an already-sorted index; reset first so we can actually call .set_index()
ddf = dask.datasets.timeseries(end="2000-07-01").reset_index().astype({"timestamp": str})
ddf = ddf.set_index("timestamp", sorted=True)
ddf.to_csv("/tmp/dask_test/")
```
pre-#8517, this finishes in ~30s on my laptop; post-#8517, it fails with
```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-4-29e339cba513> in <module>
      1 from uuid import uuid4
----> 2 ddf.to_csv(f"gs://model_bigquery_tmp/{uuid4().hex}/")

~/Dropbox/Documents/dask/dask/dataframe/core.py in to_csv(self, filename, **kwargs)
   1621         from .io import to_csv
   1622 
-> 1623         return to_csv(self, filename, **kwargs)
   1624 
   1625     def to_sql(

~/Dropbox/Documents/dask/dask/dataframe/io/csv.py in to_csv(df, filename, single_file, encoding, mode, name_function, compression, compute, scheduler, storage_options, header_first_partition_only, compute_kwargs, **kwargs)
    955         import dask
    956 
--> 957         return list(dask.compute(*values, **compute_kwargs))
    958     else:
    959         return values

~/Dropbox/Documents/dask/dask/base.py in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    573         postcomputes.append(x.__dask_postcompute__())
    574 
--> 575     results = schedule(dsk, keys, **kwargs)
    576     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    577 

~/Dropbox/Documents/distributed/distributed/client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
   3002                     should_rejoin = False
   3003             try:
-> 3004                 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
   3005             finally:
   3006                 for f in futures.values():

~/Dropbox/Documents/distributed/distributed/client.py in gather(self, futures, errors, direct, asynchronous)
   2176             else:
   2177                 local_worker = None
-> 2178             return self.sync(
   2179                 self._gather,
   2180                 futures,

~/Dropbox/Documents/distributed/distributed/utils.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    307             return future
    308         else:
--> 309             return sync(
    310                 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
    311             )

~/Dropbox/Documents/distributed/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    374     if error:
    375         typ, exc, tb = error
--> 376         raise exc.with_traceback(tb)
    377     else:
    378         return result

~/Dropbox/Documents/distributed/distributed/utils.py in f()
    347                 future = asyncio.wait_for(future, callback_timeout)
    348             future = asyncio.ensure_future(future)
--> 349             result = yield future
    350         except Exception:
    351             error = sys.exc_info()

~/model/.venv/lib/python3.9/site-packages/tornado/gen.py in run(self)
    760 
    761                     try:
--> 762                         value = future.result()
    763                     except Exception:
    764                         exc_info = sys.exc_info()

~/Dropbox/Documents/distributed/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
   2039                             exc = CancelledError(key)
   2040                         else:
-> 2041                             raise exception.with_traceback(traceback)
   2042                         raise exc
   2043                     if errors == "skip":

Exception: RecursionError('maximum recursion depth exceeded in __instancecheck__')
```
and also eats up an enormous amount of memory in the process (reaches several times higher than peak memory in the older version before giving up).

I haven't been able to reproduce the same error without the `distributed.Client()`, the computation seems to just hang forever with other schedulers...will keep playing around with the size of dataset/partitions and see if I can get something breaking w/o `distributed`.

EDIT: it did finally complete after ~10 minutes with `scheduler="threads"` (compared to ~90s on 2022.2.1), no recursion error
EDIT 2: couldn't reproduce the error w/ `processes` but I did manage to brick my OS by using so much memory 😅 

cc @jsignell @gjoseph92 


Discussion:
@jsignell do you have time to look at this?  No worries if not.
Hey @bnaul thanks for the report and bisecting the issue. I'll be taking a look at this in the coming days.
@jsignell do you have time to look at this?  No worries if not.
Hey @bnaul thanks for the report and bisecting the issue. I'll be taking a look at this in the coming days.

Resolving patch:
diff --git a/dask/dataframe/shuffle.py b/dask/dataframe/shuffle.py
--- a/dask/dataframe/shuffle.py
+++ b/dask/dataframe/shuffle.py
@@ -987,17 +987,17 @@ def fix_overlap(ddf, mins, maxes, lens):
     # drop empty partitions by mapping each partition in a new graph to a particular
     # partition on the old graph.
     dsk = {(name, i): (ddf._name, div) for i, div in enumerate(non_empties)}
+    ddf_keys = list(dsk.values())
     divisions = tuple(mins) + (maxes[-1],)
 
     overlap = [i for i in range(1, len(mins)) if mins[i] >= maxes[i - 1]]
 
     frames = []
     for i in overlap:
-
         # `frames` is a list of data from previous partitions that we may want to
         # move to partition i.  Here, we add "overlap" from the previous partition
         # (i-1) to this list.
-        frames.append((get_overlap, dsk[(name, i - 1)], divisions[i]))
+        frames.append((get_overlap, ddf_keys[i - 1], divisions[i]))
 
         # Make sure that any data added from partition i-1 to `frames` is removed
         # from partition i-1.
@@ -1008,7 +1008,7 @@ def fix_overlap(ddf, mins, maxes, lens):
         # to the next partition (i+1) anyway.  If we concatenate data too early,
         # we may lose rows (https://github.com/dask/dask/issues/6972).
         if i == len(mins) - 2 or divisions[i] != divisions[i + 1]:
-            frames.append(dsk[(name, i)])
+            frames.append(ddf_keys[i])
             dsk[(name, i)] = (methods.concat, frames)
             frames = []
 


Test patch:
diff --git a/dask/tests/test_distributed.py b/dask/tests/test_distributed.py
--- a/dask/tests/test_distributed.py
+++ b/dask/tests/test_distributed.py
@@ -744,3 +744,18 @@ def __init__(self, val):
     )
 
     assert (await c.compute(result)).val == 170
+
+
+def test_set_index_no_resursion_error(c):
+    # see: https://github.com/dask/dask/issues/8955
+    pytest.importorskip("dask.dataframe")
+    try:
+        ddf = (
+            dask.datasets.timeseries(start="2000-01-01", end="2000-07-01", freq="12h")
+            .reset_index()
+            .astype({"timestamp": str})
+        )
+        ddf = ddf.set_index("timestamp", sorted=True)
+        ddf.compute()
+    except RecursionError:
+        pytest.fail("dd.set_index triggered a recursion error")