Repository: dask/distributed

Issue:
Actors on same worker cause deadlock
Consider
```python
client = dask.distributed.Client()

class Minimal:
    
    def thetime(self):
        import time
        return time.time()

class UsesMinimal():
     
     def getthetime(self, ac):
         return ac.thetime().result()
```

The following works
```python
ac = client.submit(Minimal, actor=True, workers=["127.0.0.1:61047"]).result()
ac2 = client.submit(UsesMinimal, actor=True, workers=['tcp://127.0.0.1:61046']).result()
ac2.getthetime(ac).result()
```

but if they are on the same worker
```python
ac2 = client.submit(UsesMinimal, actor=True, workers=['tcp://127.0.0.1:61047']).result()
ac2.getthetime(ac).result()
```
... deadlock

- Dask version: master (dask/distributed)
- Python version: 3.7
- Operating System: osx
- Install method (conda, pip, source): source



Discussion:
Actors use the standard thread pool used by default.  If that worker has
only one thread then one actor will be waiting for the other to finish.  In
the actor-calling-actor state we probably want to secede/rejoin around the
call.

On Fri, Nov 6, 2020 at 7:28 AM Martin Durant <notifications@github.com>
wrote:

> Consider
>
> client = dask.distributed.Client()
> class Minimal:
>
>     def thetime(self):
>         import time
>         return time.time()
> class UsesMinimal():
>
>      def getthetime(self, ac):
>          return ac.thetime().result()
>
> The following works
>
> ac = client.submit(Minimal, actor=True, workers=["127.0.0.1:61047"]).result()ac2 = client.submit(UsesMinimal, actor=True, workers=['tcp://127.0.0.1:61046']).result()ac2.getthetime(ac).result()
>
> but if they are on the same worker
>
> ac2 = client.submit(UsesMinimal, actor=True, workers=['tcp://127.0.0.1:61047']).result()ac2.getthetime(ac).result()
>
> ... deadlock
>
>    - Dask version: master (dask/distributed)
>    - Python version: 3.7
>    - Operating System: osx
>    - Install method (conda, pip, source): source
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/4224>, or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AACKZTBPVVK4ILYHYAZPZ5TSOQIZRANCNFSM4TMZ4BNQ>
> .
>

Would you think it better to phrase this in async style? I wasn't sure how to do that - `return await ac.thetime()` ?
I tried the following
```
class UsesMinimal():
    def __init__(self):
        try:
            dask.distributed.get_worker()
            self.worker = True
        except:
            self.worker = False

    def getthetime(self, ac):
        if self.worker:
            dask.distributed.secede()
        r = ac.thetime().result()
        if self.worker:
            dask.distributed.rejoin()
        return r
```
and still deadlocks.

-edit- sometimes I get bad errors like
```
tornado.application - ERROR - Exception in callback <bound method Worker.trigger_profile of <Worker: 'tcp://127.0.0.1:62092', 0, Status.running, stored: 0, running: 0/12, ready: 0, comm: 0, waiting: 0>>
Traceback (most recent call last):
  File "/Users/mdurant/conda/envs/py37/lib/python3.7/site-packages/tornado/ioloop.py", line 907, in _run
    return self.callback()
  File "/Users/mdurant/conda/envs/py37/lib/python3.7/site-packages/distributed/worker.py", line 2734, in trigger_profile
    frames = {ident: frames[ident] for ident in active_threads}
  File "/Users/mdurant/conda/envs/py37/lib/python3.7/site-packages/distributed/worker.py", line 2734, in <dictcomp>
    frames = {ident: frames[ident] for ident in active_threads}
KeyError: 123145618325504
```
This works
```python
class Minimal:

    async def thetime(self):
        import time
        return time.time()

class UsesMinimal():

    async def getthetime(self, ac):
        import time
        t0 = time.time()
        return await ac.thetime() - t0

ac = client.submit(Minimal, actor=True).result()
ac2 = client.submit(UsesMinimal, actor=True).result()
%time ac2.getthetime(ac).result()
CPU times: user 2.99 ms, sys: 1.16 ms, total: 4.15 ms
Wall time: 4.7 ms
0.0005402565002441406
```

(0.54ms latency to call the actor from an actor on the same worker and wait for the result)
Async will definitely avoid the deadlock.  If same-worker actors are going to be common then this is probably something that we could special-case pretty easily and get down to normal Python speeds.

This would probably also avoid the deadlock, because you could just call the object directly in the same thread.

Resolving patch:
diff --git a/distributed/actor.py b/distributed/actor.py
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -1,11 +1,12 @@
 import asyncio
 import functools
+from inspect import iscoroutinefunction
 import threading
 from queue import Queue
 
 from .client import Future, default_client
 from .protocol import to_serialize
-from .utils import sync
+from .utils import thread_state, sync
 from .utils_comm import WrappedKey
 from .worker import get_worker
 
@@ -118,13 +119,33 @@ def __dir__(self):
         return sorted(o)
 
     def __getattr__(self, key):
-        attr = getattr(self._cls, key)
 
         if self._future and self._future.status not in ("finished", "pending"):
             raise ValueError(
                 "Worker holding Actor was lost.  Status: " + self._future.status
             )
 
+        if (
+            self._worker
+            and self._worker.address == self._address
+            and getattr(thread_state, "actor", False)
+        ):
+            # actor calls actor on same worker
+            actor = self._worker.actors[self.key]
+            attr = getattr(actor, key)
+
+            if iscoroutinefunction(attr):
+                return attr
+
+            elif callable(attr):
+                return lambda *args, **kwargs: ActorFuture(
+                    None, None, result=attr(*args, **kwargs)
+                )
+            else:
+                return attr
+
+        attr = getattr(self._cls, key)
+
         if callable(attr):
 
             @functools.wraps(attr)
@@ -206,9 +227,14 @@ class ActorFuture:
     Actor
     """
 
-    def __init__(self, q, io_loop):
+    def __init__(self, q, io_loop, result=None):
         self.q = q
         self.io_loop = io_loop
+        if result:
+            self._cached_result = result
+
+    def __await__(self):
+        return self.result()
 
     def result(self, timeout=None):
         try:
diff --git a/distributed/worker.py b/distributed/worker.py
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3416,6 +3416,7 @@ def apply_function_actor(
 
     thread_state.execution_state = execution_state
     thread_state.key = key
+    thread_state.actor = True
 
     result = function(*args, **kwargs)
 


Test patch:
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -6,7 +6,7 @@
 
 import dask
 from distributed import Actor, ActorFuture, Client, Future, wait, Nanny
-from distributed.utils_test import gen_cluster
+from distributed.utils_test import cluster, gen_cluster
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 from distributed.metrics import time
 
@@ -21,11 +21,25 @@ def increment(self):
         self.n += 1
         return self.n
 
+    async def ainc(self):
+        self.n += 1
+        return self.n
+
     def add(self, x):
         self.n += x
         return self.n
 
 
+class UsesCounter:
+    # An actor whose method argument is another actor
+
+    def do_inc(self, ac):
+        return ac.increment().result()
+
+    async def ado_inc(self, ac):
+        return await ac.ainc()
+
+
 class List:
     L = []
 
@@ -550,3 +564,20 @@ async def wait(self):
     await waiter.set()
 
     await c.gather(futures)
+
+
+def test_one_thread_deadlock():
+    with cluster(nworkers=2) as (cl, w):
+        client = Client(cl["address"])
+        ac = client.submit(Counter, actor=True).result()
+        ac2 = client.submit(UsesCounter, actor=True, workers=[ac._address]).result()
+
+        assert ac2.do_inc(ac).result() == 1
+
+
+@gen_cluster(client=True)
+async def test_async_deadlock(client, s, a, b):
+    ac = await client.submit(Counter, actor=True)
+    ac2 = await client.submit(UsesCounter, actor=True, workers=[ac._address])
+
+    assert (await ac2.ado_inc(ac)) == 1