Repository: dask/distributed

Issue:
Attribute error when calling SpecCluster.adapt()
When calling `cluster.adapt()` on a `SpecCluster` (in this case `FargateCluster` from jacobtomlinson/dask-cloud) I'm getting a traceback error where the adaptive code is trying to talk to the scheduler. 

I think this is a confusion between `scheduler` and `scheduler_comm` in `SpecCluster`. But as the call is not within `SpecCluster` but a level higher in `Cluster` I'm not initially sure how to resolve.

cc @mrocklin 

```python-traceback
In [3]: cluster.adapt(minimum=1, maximum=20)                                                                                              
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-3-a0f85347b68c> in <module>
----> 1 cluster.adapt(minimum=1, maximum=20)

~/Projects/dask/distributed/distributed/deploy/cluster.py in adapt(self, **kwargs)
     77             self._adaptive_options = {}
     78         self._adaptive_options.update(kwargs)
---> 79         self._adaptive = Adaptive(self.scheduler, self, **self._adaptive_options)
     80         return self._adaptive
     81 

~/Projects/dask/distributed/distributed/deploy/adaptive.py in __init__(self, scheduler, cluster, interval, startup_cost, scale_factor, minimum, maximum, wait_count, target_duration, worker_key, **kwargs)
    120         self.target_duration = parse_timedelta(target_duration)
    121 
--> 122         self.scheduler.handlers["adaptive_recommendations"] = self.recommendations
    123 
    124     def stop(self):

AttributeError: 'Scheduler' object has no attribute 'handlers'

In [4]: tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <tornado.platform.asyncio.AsyncIOLoop object at 0x7ff704708898>>, <Future finished exception=AttributeError("'Scheduler' object has no attribute 'adaptive_target'")>)
Traceback (most recent call last):
  File "/home/nfs/jtomlinson/miniconda3/envs/dask-csp/lib/python3.7/site-packages/tornado/ioloop.py", line 743, in _run_callback
    ret = callback()
  File "/home/nfs/jtomlinson/miniconda3/envs/dask-csp/lib/python3.7/site-packages/tornado/ioloop.py", line 767, in _discard_future_result
    future.result()
  File "/home/nfs/jtomlinson/miniconda3/envs/dask-csp/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
    yielded = next(result)
  File "/home/nfs/jtomlinson/Projects/dask/distributed/distributed/deploy/adaptive.py", line 259, in _adapt
    recommendations = self.recommendations()
  File "/home/nfs/jtomlinson/Projects/dask/distributed/distributed/deploy/adaptive.py", line 209, in recommendations
    n = self.scheduler.adaptive_target(target_duration=self.target_duration)
AttributeError: 'Scheduler' object has no attribute 'adaptive_target'
```


Discussion:
Yup.  It looks like this bit of the code is still assuming that the
scheduler is local.  Instead of directly mutating the scheduler.handlers
state we probably need to either send some signal up to the scheduler, or
have this on all the time.  I'll take a look, but possibly not before early
next week.

On Fri, Jul 19, 2019 at 4:55 AM Jacob Tomlinson <notifications@github.com>
wrote:

> When calling cluster.adapt() on a SpecCluster (in this case FargateCluster
> from jacobtomlinson/dask-cloud) I'm getting a traceback error where the
> adaptive code is trying to talk to the scheduler.
>
> I think this is a confusion between scheduler and scheduler_comm in
> SpecCluster. But as the call is not within SpecCluster but a level higher
> in Cluster I'm not initially sure how to resolve.
>
> cc @mrocklin <https://github.com/mrocklin>
>
> In [3]: cluster.adapt(minimum=1, maximum=20)
> ---------------------------------------------------------------------------
> AttributeError                            Traceback (most recent call last)
> <ipython-input-3-a0f85347b68c> in <module>
> ----> 1 cluster.adapt(minimum=1, maximum=20)
>
> ~/Projects/dask/distributed/distributed/deploy/cluster.py in adapt(self, **kwargs)
>      77             self._adaptive_options = {}
>      78         self._adaptive_options.update(kwargs)
> ---> 79         self._adaptive = Adaptive(self.scheduler, self, **self._adaptive_options)
>      80         return self._adaptive
>      81
>
> ~/Projects/dask/distributed/distributed/deploy/adaptive.py in __init__(self, scheduler, cluster, interval, startup_cost, scale_factor, minimum, maximum, wait_count, target_duration, worker_key, **kwargs)
>     120         self.target_duration = parse_timedelta(target_duration)
>     121
> --> 122         self.scheduler.handlers["adaptive_recommendations"] = self.recommendations
>     123
>     124     def stop(self):
> AttributeError: 'Scheduler' object has no attribute 'handlers'
>
> In [4]: tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <tornado.platform.asyncio.AsyncIOLoop object at 0x7ff704708898>>, <Future finished exception=AttributeError("'Scheduler' object has no attribute 'adaptive_target'")>)
> Traceback (most recent call last):
>   File "/home/nfs/jtomlinson/miniconda3/envs/dask-csp/lib/python3.7/site-packages/tornado/ioloop.py", line 743, in _run_callback
>     ret = callback()
>   File "/home/nfs/jtomlinson/miniconda3/envs/dask-csp/lib/python3.7/site-packages/tornado/ioloop.py", line 767, in _discard_future_result
>     future.result()
>   File "/home/nfs/jtomlinson/miniconda3/envs/dask-csp/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
>     yielded = next(result)
>   File "/home/nfs/jtomlinson/Projects/dask/distributed/distributed/deploy/adaptive.py", line 259, in _adapt
>     recommendations = self.recommendations()
>   File "/home/nfs/jtomlinson/Projects/dask/distributed/distributed/deploy/adaptive.py", line 209, in recommendations
>     n = self.scheduler.adaptive_target(target_duration=self.target_duration)AttributeError: 'Scheduler' object has no attribute 'adaptive_target'
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/2858?email_source=notifications&email_token=AACKZTFAYVQFIX2DJ4MSJE3QAGTTXA5CNFSM4IFFH5HKYY3PNVWWK3TUL52HS4DFUVEXG43VMWVGG33NNVSW45C7NFSM4HAIANOQ>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AACKZTAPZ4KNHEL6SIASAATQAGTTXANCNFSM4IFFH5HA>
> .
>


Resolving patch:
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -2,12 +2,12 @@
 
 from collections import deque
 import logging
-import math
 
 from tornado import gen
 
 from ..metrics import time
 from ..utils import log_errors, PeriodicCallback, parse_timedelta
+from ..protocol import pickle
 
 logger = logging.getLogger(__name__)
 
@@ -80,14 +80,10 @@ class Adaptive(object):
     resized. The default implementation checks if there are too many tasks
     per worker or too little memory available (see :meth:`Adaptive.needs_cpu`
     and :meth:`Adaptive.needs_memory`).
-
-    :meth:`Adaptive.get_scale_up_kwargs` method controls the arguments passed to
-    the cluster's ``scale_up`` method.
     '''
 
     def __init__(
         self,
-        scheduler,
         cluster=None,
         interval="1s",
         startup_cost="1s",
@@ -96,20 +92,19 @@ def __init__(
         maximum=None,
         wait_count=3,
         target_duration="5s",
-        worker_key=lambda x: x,
+        worker_key=None,
         **kwargs
     ):
         interval = parse_timedelta(interval, default="ms")
         self.worker_key = worker_key
-        self.scheduler = scheduler
         self.cluster = cluster
         self.startup_cost = parse_timedelta(startup_cost, default="s")
         self.scale_factor = scale_factor
         if self.cluster:
             self._adapt_callback = PeriodicCallback(
-                self._adapt, interval * 1000, io_loop=scheduler.loop
+                self._adapt, interval * 1000, io_loop=self.loop
             )
-            self.scheduler.loop.add_callback(self._adapt_callback.start)
+            self.loop.add_callback(self._adapt_callback.start)
         self._adapting = False
         self._workers_to_close_kwargs = kwargs
         self.minimum = minimum
@@ -119,7 +114,9 @@ def __init__(
         self.wait_count = wait_count
         self.target_duration = parse_timedelta(target_duration)
 
-        self.scheduler.handlers["adaptive_recommendations"] = self.recommendations
+    @property
+    def scheduler(self):
+        return self.cluster.scheduler_comm
 
     def stop(self):
         if self.cluster:
@@ -127,7 +124,7 @@ def stop(self):
             self._adapt_callback = None
             del self._adapt_callback
 
-    def workers_to_close(self, **kwargs):
+    async def workers_to_close(self, **kwargs):
         """
         Determine which, if any, workers should potentially be removed from
         the cluster.
@@ -145,73 +142,53 @@ def workers_to_close(self, **kwargs):
         --------
         Scheduler.workers_to_close
         """
-        if len(self.scheduler.workers) <= self.minimum:
+        if len(self.cluster.workers) <= self.minimum:
             return []
 
         kw = dict(self._workers_to_close_kwargs)
         kw.update(kwargs)
 
-        if self.maximum is not None and len(self.scheduler.workers) > self.maximum:
-            kw["n"] = len(self.scheduler.workers) - self.maximum
+        if self.maximum is not None and len(self.cluster.workers) > self.maximum:
+            kw["n"] = len(self.cluster.workers) - self.maximum
 
-        L = self.scheduler.workers_to_close(**kw)
-        if len(self.scheduler.workers) - len(L) < self.minimum:
-            L = L[: len(self.scheduler.workers) - self.minimum]
+        L = await self.scheduler.workers_to_close(**kw)
+        if len(self.cluster.workers) - len(L) < self.minimum:
+            L = L[: len(self.cluster.workers) - self.minimum]
 
         return L
 
-    @gen.coroutine
-    def _retire_workers(self, workers=None):
+    async def _retire_workers(self, workers=None):
         if workers is None:
-            workers = self.workers_to_close(key=self.worker_key, minimum=self.minimum)
+            workers = await self.workers_to_close(
+                key=pickle.dumps(self.worker_key) if self.worker_key else None,
+                minimum=self.minimum,
+            )
         if not workers:
             raise gen.Return(workers)
         with log_errors():
-            yield self.scheduler.retire_workers(
+            await self.scheduler.retire_workers(
                 workers=workers, remove=True, close_workers=True
             )
 
             logger.info("Retiring workers %s", workers)
             f = self.cluster.scale_down(workers)
             if hasattr(f, "__await__"):
-                yield f
-
-            raise gen.Return(workers)
-
-    def get_scale_up_kwargs(self):
-        """
-        Get the arguments to be passed to ``self.cluster.scale_up``.
-
-        Notes
-        -----
-        By default the desired number of total workers is returned (``n``).
-        Subclasses should ensure that the return dictionary includes a key-
-        value pair for ``n``, either by implementing it or by calling the
-        parent's ``get_scale_up_kwargs``.
+                await f
 
-        See Also
-        --------
-        LocalCluster.scale_up
-        """
-        target = math.ceil(self.scheduler.total_occupancy / self.target_duration)
-        instances = max(
-            1, len(self.scheduler.workers) * self.scale_factor, target, self.minimum
-        )
-
-        if self.maximum:
-            instances = min(self.maximum, instances)
+            return workers
 
-        instances = int(instances)
-        logger.info("Scaling up to %d workers", instances)
-        return {"n": instances}
-
-    def recommendations(self, comm=None):
-        n = self.scheduler.adaptive_target(target_duration=self.target_duration)
+    async def recommendations(self, comm=None):
+        n = await self.scheduler.adaptive_target(target_duration=self.target_duration)
         if self.maximum is not None:
             n = min(self.maximum, n)
         if self.minimum is not None:
             n = max(self.minimum, n)
-        workers = set(self.workers_to_close(key=self.worker_key, minimum=self.minimum))
+        workers = set(
+            await self.workers_to_close(
+                key=pickle.dumps(self.worker_key) if self.worker_key else None,
+                minimum=self.minimum,
+            )
+        )
         try:
             current = len(self.cluster.worker_spec)
         except AttributeError:
@@ -249,14 +226,13 @@ def recommendations(self, comm=None):
             self.close_counts.clear()
             return None
 
-    @gen.coroutine
-    def _adapt(self):
+    async def _adapt(self):
         if self._adapting:  # Semaphore to avoid overlapping adapt calls
             return
 
         self._adapting = True
         try:
-            recommendations = self.recommendations()
+            recommendations = await self.recommendations()
             if not recommendations:
                 return
             status = recommendations.pop("status")
@@ -264,13 +240,17 @@ def _adapt(self):
                 f = self.cluster.scale(**recommendations)
                 self.log.append((time(), "up", recommendations))
                 if hasattr(f, "__await__"):
-                    yield f
+                    await f
 
             elif status == "down":
                 self.log.append((time(), "down", recommendations["workers"]))
-                workers = yield self._retire_workers(workers=recommendations["workers"])
+                workers = await self._retire_workers(workers=recommendations["workers"])
         finally:
             self._adapting = False
 
     def adapt(self):
-        self.scheduler.loop.add_callback(self._adapt)
+        self.loop.add_callback(self._adapt)
+
+    @property
+    def loop(self):
+        return self.cluster.loop
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -62,7 +62,7 @@ def scale_down(self, workers: List[str]):
     LocalCluster: a simple implementation with local workers
     """
 
-    def adapt(self, **kwargs):
+    def adapt(self, Adaptive=Adaptive, **kwargs):
         """ Turn on adaptivity
 
         For keyword arguments see dask.distributed.Adaptive
@@ -76,7 +76,7 @@ def adapt(self, **kwargs):
         if not hasattr(self, "_adaptive_options"):
             self._adaptive_options = {}
         self._adaptive_options.update(kwargs)
-        self._adaptive = Adaptive(self.scheduler, self, **self._adaptive_options)
+        self._adaptive = Adaptive(self, **self._adaptive_options)
         return self._adaptive
 
     @property
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1070,6 +1070,7 @@ def __init__(
             "get_task_stream": self.get_task_stream,
             "register_worker_plugin": self.register_worker_plugin,
             "adaptive_target": self.adaptive_target,
+            "workers_to_close": self.workers_to_close,
         }
 
         self._transitions = {
@@ -2935,7 +2936,9 @@ def replicate(
             },
         )
 
-    def workers_to_close(self, memory_ratio=None, n=None, key=None, minimum=None):
+    def workers_to_close(
+        self, comm=None, memory_ratio=None, n=None, key=None, minimum=None
+    ):
         """
         Find workers that we can close with low cost
 
@@ -2997,6 +3000,10 @@ def workers_to_close(self, memory_ratio=None, n=None, key=None, minimum=None):
 
             if key is None:
                 key = lambda ws: ws.address
+            if isinstance(key, bytes) and dask.config.get(
+                "distributed.scheduler.pickle"
+            ):
+                key = pickle.loads(key)
 
             groups = groupby(key, self.workers.values())
 
@@ -3225,6 +3232,14 @@ def feed(
         Caution: this runs arbitrary Python code on the scheduler.  This should
         eventually be phased out.  It is mostly used by diagnostics.
         """
+        if not dask.config.get("distributed.scheduler.pickle"):
+            logger.warn(
+                "Tried to call 'feed' route with custom fucntions, but "
+                "pickle is disallowed.  Set the 'distributed.scheduler.pickle'"
+                "config value to True to use the 'feed' route (this is mostly "
+                "commonly used with progress bars)"
+            )
+            return
         import pickle
 
         interval = parse_timedelta(interval)
@@ -4730,7 +4745,7 @@ def check_idle(self):
         if close:
             self.loop.add_callback(self.close)
 
-    def adaptive_target(self, target_duration="5s"):
+    def adaptive_target(self, comm=None, target_duration="5s"):
         """ Desired number of workers based on the current workload
 
         This looks at the current running tasks and memory use, and returns a


Test patch:
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -2,33 +2,19 @@
 
 from time import sleep
 
+import pytest
 from toolz import frequencies, pluck
 from tornado import gen
 from tornado.ioloop import IOLoop
 
 from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
-from distributed.utils_test import gen_cluster, gen_test, slowinc, clean
+from distributed.utils_test import gen_test, slowinc, clean
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from distributed.metrics import time
 
 
-def test_get_scale_up_kwargs(loop):
-    with LocalCluster(
-        0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
-    ) as cluster:
-
-        alc = Adaptive(cluster.scheduler, cluster, interval=100, scale_factor=3)
-        assert alc.get_scale_up_kwargs() == {"n": 1}
-
-        with Client(cluster, loop=loop) as c:
-            future = c.submit(lambda x: x + 1, 1)
-            assert future.result() == 2
-            assert c.nthreads()
-            assert alc.get_scale_up_kwargs() == {"n": 3}
-
-
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_simultaneous_scale_up_and_down(c, s, *workers):
+@pytest.mark.asyncio
+async def test_simultaneous_scale_up_and_down():
     class TestAdaptive(Adaptive):
         def get_scale_up_kwargs(self):
             assert False
@@ -36,34 +22,35 @@ def get_scale_up_kwargs(self):
         def _retire_workers(self):
             assert False
 
-    class TestCluster(object):
+    class TestCluster(LocalCluster):
         def scale_up(self, n, **kwargs):
             assert False
 
         def scale_down(self, workers):
             assert False
 
-    cluster = TestCluster()
+    async with TestCluster(n_workers=4, processes=False, asynchronous=True) as cluster:
+        async with Client(cluster, asynchronous=True) as c:
+            s = cluster.scheduler
+            s.task_duration["a"] = 4
+            s.task_duration["b"] = 4
+            s.task_duration["c"] = 1
 
-    s.task_duration["a"] = 4
-    s.task_duration["b"] = 4
-    s.task_duration["c"] = 1
+            future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
 
-    future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
+            while len(s.rprocessing) < 3:
+                await gen.sleep(0.001)
 
-    while len(s.rprocessing) < 3:
-        yield gen.sleep(0.001)
+            ta = cluster.adapt(interval="100 ms", scale_factor=2, Adaptive=TestAdaptive)
 
-    ta = TestAdaptive(s, cluster, interval=100, scale_factor=2)
-
-    yield gen.sleep(0.3)
+            await gen.sleep(0.3)
 
 
 def test_adaptive_local_cluster(loop):
     with LocalCluster(
         0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
     ) as cluster:
-        alc = Adaptive(cluster.scheduler, cluster, interval=100)
+        alc = cluster.adapt(interval="100 ms")
         with Client(cluster, loop=loop) as c:
             assert not c.nthreads()
             future = c.submit(lambda x: x + 1, 1)
@@ -128,41 +115,34 @@ def test_adaptive_local_cluster_multi_workers():
         yield cluster.close()
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, active_rpc_timeout=10)
-def test_adaptive_scale_down_override(c, s, *workers):
+@pytest.mark.asyncio
+async def test_adaptive_scale_down_override():
     class TestAdaptive(Adaptive):
         def __init__(self, *args, **kwargs):
             self.min_size = kwargs.pop("min_size", 0)
             Adaptive.__init__(self, *args, **kwargs)
 
-        def workers_to_close(self, **kwargs):
-            num_workers = len(self.scheduler.workers)
-            to_close = self.scheduler.workers_to_close(**kwargs)
+        async def workers_to_close(self, **kwargs):
+            num_workers = len(self.cluster.workers)
+            to_close = await self.scheduler.workers_to_close(**kwargs)
             if num_workers - len(to_close) < self.min_size:
                 to_close = to_close[: num_workers - self.min_size]
 
             return to_close
 
-    class TestCluster(object):
+    class TestCluster(LocalCluster):
         def scale_up(self, n, **kwargs):
             assert False
 
-        def scale_down(self, workers):
-            assert False
-
-        @property
-        def workers(self):
-            return s.workers
-
-    assert len(s.workers) == 10
-
-    # Assert that adaptive cycle does not reduce cluster below minimum size
-    # as determined via override.
-    cluster = TestCluster()
-    ta = TestAdaptive(s, cluster, min_size=2, interval=0.1, scale_factor=2)
-    yield gen.sleep(0.3)
+    async with TestCluster(n_workers=10, processes=False, asynchronous=True) as cluster:
+        ta = cluster.adapt(
+            min_size=2, interval=0.1, scale_factor=2, Adaptive=TestAdaptive
+        )
+        await gen.sleep(0.3)
 
-    assert len(s.workers) == 2
+        # Assert that adaptive cycle does not reduce cluster below minimum size
+        # as determined via override.
+        assert len(cluster.scheduler.workers) == 2
 
 
 @gen_test()
@@ -176,14 +156,7 @@ def test_min_max():
         asynchronous=True,
     )
     try:
-        adapt = Adaptive(
-            cluster.scheduler,
-            cluster,
-            minimum=1,
-            maximum=2,
-            interval="20 ms",
-            wait_count=10,
-        )
+        adapt = cluster.adapt(minimum=1, maximum=2, interval="20 ms", wait_count=10)
         c = yield Client(cluster, asynchronous=True)
 
         start = time()
@@ -237,7 +210,7 @@ def test_avoid_churn():
     )
     client = yield Client(cluster, asynchronous=True)
     try:
-        adapt = Adaptive(cluster.scheduler, cluster, interval="20 ms", wait_count=5)
+        adapt = cluster.adapt(interval="20 ms", wait_count=5)
 
         for i in range(10):
             yield client.submit(slowinc, i, delay=0.040)
@@ -267,7 +240,7 @@ def test_adapt_quickly():
         dashboard_address=None,
     )
     client = yield Client(cluster, asynchronous=True)
-    adapt = Adaptive(cluster.scheduler, cluster, interval=20, wait_count=5, maximum=10)
+    adapt = cluster.adapt(interval=20, wait_count=5, maximum=10)
     try:
         future = client.submit(slowinc, 1, delay=0.100)
         yield wait(future)
@@ -346,9 +319,7 @@ def test_no_more_workers_than_tasks():
     )
     yield cluster._start()
     try:
-        adapt = Adaptive(
-            cluster.scheduler, cluster, minimum=0, maximum=4, interval="10 ms"
-        )
+        adapt = cluster.adapt(minimum=0, maximum=4, interval="10 ms")
         client = yield Client(cluster, asynchronous=True, loop=loop)
         cluster.scheduler.task_duration["slowinc"] = 1000