Repository: modin-project/modin

Issue:
Tuning `Dask` engine to operate with big datasets
Sometimes while using Modin with `Dask` engine, different recursive errors with non-informative messages can occur, see reproducer below:

<details>
<summary><b>Reproducer</b></summary>

Reproducer code:
```
import os
from timeit import default_timer as timer

import modin.config

columns_names = [
    "trip_id",
    "vendor_id",
    "pickup_datetime",
    "dropoff_datetime",
    "store_and_fwd_flag",
    "rate_code_id",
    "pickup_longitude",
    "pickup_latitude",
    "dropoff_longitude",
    "dropoff_latitude",
    "passenger_count",
    "trip_distance",
    "fare_amount",
    "extra",
    "mta_tax",
    "tip_amount",
    "tolls_amount",
    "ehail_fee",
    "improvement_surcharge",
    "total_amount",
    "payment_type",
    "trip_type",
    "pickup",
    "dropoff",
    "cab_type",
    "precipitation",
    "snow_depth",
    "snowfall",
    "max_temperature",
    "min_temperature",
    "average_wind_speed",
    "pickup_nyct2010_gid",
    "pickup_ctlabel",
    "pickup_borocode",
    "pickup_boroname",
    "pickup_ct2010",
    "pickup_boroct2010",
    "pickup_cdeligibil",
    "pickup_ntacode",
    "pickup_ntaname",
    "pickup_puma",
    "dropoff_nyct2010_gid",
    "dropoff_ctlabel",
    "dropoff_borocode",
    "dropoff_boroname",
    "dropoff_ct2010",
    "dropoff_boroct2010",
    "dropoff_cdeligibil",
    "dropoff_ntacode",
    "dropoff_ntaname",
    "dropoff_puma",
]

kwargs = dict(
    names=columns_names,
    header=None,
    nrows=None,
    parse_dates=["pickup_datetime", "dropoff_datetime"],
)

def meas(iters, func, *args, **kwargs):
    t_res = float("inf")
    for _ in range(iters):
        t0 = timer()
        ans = func(*args, **kwargs)
        t = timer() - t0
        t_res = t if t < t_res else t
    
    return ans, t_res

def func_read_csv(data_file, **kwargs):
    return pd.read_csv(data_file, **kwargs)

def func_q1(df):
    return df.groupby("cab_type").size()

def func_q2(df):
    return df.groupby("passenger_count").agg({"total_amount": "mean"})

if __name__ == '__main__':
    # from distributed import Client
    # client = Client(n_workers=modin.config.CpuCount.get(), memory_limit="64GB")

    import modin.pandas as pd

    # 20M rows file
    data_file = "/trips_xaa.csv"
    modin.config.Engine.put("Dask")
    df, t_read_csv = meas(iters, func_read_csv, data_file, **kwargs)
    _ = func_q1(df)
    _, t_q1 = meas(iters, func_q1, df)
    _, t_q2 = meas(iters, func_q2, df)


    print(f"t_read_csv = {t_read_csv}, t_q1 = {t_q1}, t_q2 = {t_q2}")
```
Output (recursive):
```
Traceback (most recent call last):
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/worker.py", line 1988, in gather_dep
    self.rpc, deps, worker, who=self.address
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/worker.py", line 3263, in get_data_from_worker
    return await retry_operation(_get_data, operation="get_data_from_worker")
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/utils_comm.py", line 390, in retry_operation
    operation=operation,
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/utils_comm.py", line 370, in retry
    return await coro()
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/worker.py", line 3240, in _get_data
    comm = await rpc.connect(worker)
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/core.py", line 1013, in connect
    **self.connection_args,
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/comm/core.py", line 245, in connect
    _raise(error)
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/comm/core.py", line 215, in _raise
    raise IOError(msg)
OSError: Timed out trying to connect to 'tcp://127.0.0.1:39449' after 10 s: Timed out trying to connect to 'tcp://127.0.0.1:39449' after 10 s: connect() didn't finish in time
distributed.nanny - WARNING - Worker exceeded 95% memory budget. Restarting
Traceback (most recent call last):
  File "dask_memory.py", line 119, in <module>
distributed.nanny - WARNING - Restarting worker
    _, t_q2 = meas(iters, func_q2, df)
  File "dask_memory.py", line 90, in meas
    ans = func(*args, **kwargs)
  File "dask_memory.py", line 103, in func_q2
    return df.groupby("passenger_count").agg({"total_amount": "mean"})
  File "/modin/modin/pandas/groupby.py", line 442, in aggregate
    **kwargs,
  File "/modin/modin/pandas/groupby.py", line 921, in _apply_agg_function
    drop=self._drop,
  File "/modin/modin/backends/pandas/query_compiler.py", line 2869, in groupby_agg
    enumerate_partitions=True,
  File "/modin/modin/engines/base/frame/data.py", line 1818, in broadcast_apply_full_axis
    for i, new_axis in enumerate([new_index, new_columns])
  File "/modin/modin/engines/base/frame/data.py", line 1818, in <listcomp>
    for i, new_axis in enumerate([new_index, new_columns])
  File "/modin/modin/engines/base/frame/data.py", line 307, in _compute_axis_labels
    axis, partitions, lambda df: df.axes[axis]
  File "/modin/modin/engines/dask/pandas_on_dask/frame/partition_manager.py", line 126, in get_indices
    new_idx = client.gather(new_idx)
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/client.py", line 1982, in gather
    asynchronous=asynchronous,
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/client.py", line 832, in sync
    self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/utils.py", line 339, in sync
    raise exc.with_traceback(tb)
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/utils.py", line 323, in f
    result[0] = yield future
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/tornado/gen.py", line 762, in run
    value = future.result()
  File "/miniconda3/envs/modin_test/lib/python3.7/site-packages/distributed/client.py", line 1841, in _gather
    raise exception.with_traceback(traceback)
distributed.scheduler.KilledWorker: ('deploy_dask_func-b0b21c69-2264-4a61-b20e-acc9b0dec748', <Worker 'tcp://127.0.0.1:38983', name: 86, memory: 0, processing: 1>)
distributed.nanny - WARNING - Restarting worker
distributed.nanny - WARNING - Restarting worker
distributed.nanny - WARNING - Restarting worker
distributed.nanny - WARNING - Restarting worker
distributed.nanny - WARNING - Restarting worker
```
</details>

This error can be caused by absence of memory on the `Dask` workers, to workaround this problem, memory amount can be increased in the next way (add this code before Modin import):
```
import modin.config
from distributed import Client
client = Client(n_workers=modin.config.CpuCount.get(), memory_limit="64GB")

import modin.pandas as pd
```



Discussion:
@amyskov Do we need to set `memory_limit` by default in the same way we do for Ray?
> @amyskov Do we need to set `memory_limit` by default in the same way we do for Ray?

Yes, i think, it is good idea, will reclassify this ticket into task.
Let's try with latest Dask first
I see this issue with a big dataset supplied to modin/examples/docker/nyc-taxi.py
```
Reading: 23.283782482147217 sec
Q1: 4.772326946258545 sec
distributed.worker - WARNING - Unmanaged memory use is high. This may indicate a memory leak or the memory may not be released to the OS; see https://distributed.dask.org/en/latest/worker.html#memtrim for more information. -- Unmanaged m
emory: 3.34 GiB -- Worker memory limit: 4.71 GiB
distributed.worker - WARNING - Unmanaged memory use is high. This may indicate a memory leak or the memory may not be released to the OS; see https://distributed.dask.org/en/latest/worker.html#memtrim for more information. -- Unmanaged m
emory: 3.31 GiB -- Worker memory limit: 4.71 GiB
distributed.nanny - WARNING - Worker exceeded 95% memory budget. Restarting
distributed.worker - ERROR - failed during get data with tcp://127.0.0.1:34109 -> tcp://127.0.0.1:33969
Traceback (most recent call last):
  File "/home/amalakho/local/miniconda3/envs/modin/lib/python3.7/site-packages/tornado/iostream.py", line 867, in _read_to_buffer
    bytes_read = self.read_from_fd(buf)
  File "/home/amalakho/local/miniconda3/envs/modin/lib/python3.7/site-packages/tornado/iostream.py", line 1140, in read_from_fd
    return self.socket.recv_into(buf, len(buf))
ConnectionResetError: [Errno 104] Connection reset by peer

```

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -170,7 +170,14 @@ class GpuCount(EnvironmentVariable, type=int):
 
 
 class Memory(EnvironmentVariable, type=int):
-    """How much memory give to each Ray worker (in bytes)."""
+    """
+    How much memory (in bytes) give to an execution engine.
+
+    Notes
+    -----
+    * In Ray case: the amount of memory to start the Plasma object store with.
+    * In Dask case: the amount of memory that is given to each worker depending on CPUs used.
+    """
 
     varname = "MODIN_MEMORY"
 
diff --git a/modin/engines/dask/utils.py b/modin/engines/dask/utils.py
--- a/modin/engines/dask/utils.py
+++ b/modin/engines/dask/utils.py
@@ -13,7 +13,7 @@
 
 """Module houses utility function to initialize Dask environment."""
 
-from modin.config import CpuCount, NPartitions
+from modin.config import CpuCount, Memory, NPartitions
 from modin.error_message import ErrorMessage
 
 
@@ -35,7 +35,10 @@ def initialize_dask():
     client = Client()
 """,
         )
-        client = Client(n_workers=CpuCount.get())
+        num_cpus = CpuCount.get()
+        memory_limit = Memory.get()
+        worker_memory_limit = memory_limit // num_cpus if memory_limit else "auto"
+        client = Client(n_workers=num_cpus, memory_limit=worker_memory_limit)
 
     num_cpus = len(client.ncores())
     NPartitions._put(num_cpus)


Test patch: