Repository: mars-project/mars

Issue:
[BUG] ActorCaller.call is at risk got hang!
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
ActorCaller._listen is at risk and will not remake related futures after the client closed. It will happen when the client.recv got successfully returned with the client closed.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and pandas
4. Full stack of the error.
5. Minimized code to reproduce the error.

code:
```python
import asyncio
import pytest
import mock

from ..backends.router import Router
from ..backends.core import ActorCaller
from ..errors import ServerClosed

@pytest.mark.asyncio
@mock.patch.object(Router, "get_client")
async def test_send_when_close(fake_get_client):

    class FakeClient:
        def __init__(self):
            self.closed = False
            self.send_num = 0
            self._messages = asyncio.Queue()
            self.dest_address = "test"

        async def send(self, message):
            await self._messages.put(message)
            self.send_num += 1
            if self.send_num >= 3:
                raise ConnectionError("test")

        async def recv(self, *args, **kwargs):
            await asyncio.sleep(3)
            res = await self._messages.get()
            return res

        async def close(self):
            self.closed = True

    fake_client = FakeClient()
    fake_get_client.side_effect = lambda *args, **kwargs: fake_client

    class FakeMessage:
        def __init__(self, id_num):
            self.message_id = id_num

    caller = ActorCaller()

    router = Router(
        external_addresses = ["test1"],
        local_address = "test2",
    )
    futures = []
    for index in range(2):
        futures.append(await caller.call(
            router = router,
            dest_address = "test1",
            message = FakeMessage(index),
            wait = False))

    with pytest.raises(ServerClosed):
        # Just wait _list run.
        await asyncio.sleep(1)
        await caller.call(
            router = router,
            dest_address = "test1",
            message = FakeMessage(2),
            wait = False)

    res0 = await futures[0]
    assert res0.message_id == 0

    with pytest.raises(ServerClosed):
        # This code will hand in here.
        await futures[1] 
```

**Expected behavior**
A clear and concise description of what you expected to happen.

**Additional context**
Add any other context about the problem here.



Discussion:

Resolving patch:
diff --git a/ci/importcheck.py b/ci/importcheck.py
--- a/ci/importcheck.py
+++ b/ci/importcheck.py
@@ -1,4 +1,18 @@
 #!/usr/bin/env python
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import ast
 import os
 import sys
diff --git a/mars/oscar/backends/core.py b/mars/oscar/backends/core.py
--- a/mars/oscar/backends/core.py
+++ b/mars/oscar/backends/core.py
@@ -68,6 +68,12 @@ async def _listen(self, client: Client):
                 for future in message_futures.values():
                     future.set_exception(e)
 
+        message_futures = self._client_to_message_futures.get(client)
+        self._client_to_message_futures[client] = dict()
+        error = ServerClosed(f'Remote server {client.dest_address} closed')
+        for future in message_futures.values():
+            future.set_exception(error)
+
     async def call(self,
                    router: Router,
                    dest_address: str,


Test patch:
diff --git a/mars/oscar/tests/test_actorcaller.py b/mars/oscar/tests/test_actorcaller.py
new file mode 100644
--- /dev/null
+++ b/mars/oscar/tests/test_actorcaller.py
@@ -0,0 +1,82 @@
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+
+import pytest
+
+from mars.oscar.backends.router import Router
+from mars.oscar.backends.core import ActorCaller
+from mars.oscar.errors import ServerClosed
+from mars.tests.core import mock
+
+
+@pytest.mark.asyncio
+@mock.patch.object(Router, "get_client")
+async def test_send_when_close(fake_get_client):
+
+    class FakeClient:
+        def __init__(self):
+            self.closed = False
+            self.send_num = 0
+            self._messages = asyncio.Queue()
+            self.dest_address = "test"
+
+        async def send(self, message):
+            await self._messages.put(message)
+            self.send_num += 1
+            if self.send_num >= 3:
+                raise ConnectionError("test")
+
+        async def recv(self, *args, **kwargs):
+            await asyncio.sleep(3)
+            res = await self._messages.get()
+            return res
+
+        async def close(self):
+            self.closed = True
+
+    fake_client = FakeClient()
+    fake_get_client.side_effect = lambda *args, **kwargs: fake_client
+
+    class FakeMessage:
+        def __init__(self, id_num):
+            self.message_id = id_num
+
+    caller = ActorCaller()
+
+    router = Router(
+        external_addresses=["test1"],
+        local_address="test2",
+    )
+    futures = []
+    for index in range(2):
+        futures.append(await caller.call(
+            router=router, dest_address="test1", message=FakeMessage(index),
+            wait=False
+        ))
+
+    with pytest.raises(ServerClosed):
+        # Just wait _list run.
+        await asyncio.sleep(1)
+        await caller.call(
+            router=router, dest_address="test1", message=FakeMessage(2),
+            wait=False
+        )
+
+    res0 = await futures[0]
+    assert res0.message_id == 0
+
+    with pytest.raises(ServerClosed):
+        await futures[1]