Repository: mars-project/mars

Issue:
[BUG] Operand runs forever when error message can't be pickled
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Operand raised an error which can't be pickled, the status of the operand is still `CALCULATING` and never convert to `FAILED`.

``` Python
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03T05:21:04Z (<ThreadPoolWorker at 0x7f14c2e8e7e0 thread_ident=0x7f14cfe3f700 threadpool-hub=<Hub at 0x7f14c338d1d0 thread_ident=0x7f14f8070740>>, <cyfunction GeventThreadPool._wrap_watch.<locals>.inner at 0x7f14c2ec37a0>) failed with ConnectionError

2021-02-03 13:21:04,748 mars.worker.calc 77 ERROR    Unexpected exception occurred in BaseCalcActor._calc_results. graph_key=402439ba636c37f11bea13fe28ad9bfb
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 383, in _wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 200, in _calc_results
    chunk_targets, retval=False).result()
  File "src/gevent/event.py", line 383, in gevent._gevent_cevent.AsyncResult.result
  File "src/gevent/event.py", line 305, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 335, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 323, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 303, in gevent._gevent_cevent.AsyncResult._raise_exception
  File "/opt/conda/lib/python3.7/site-packages/gevent/_compat.py", line 65, in reraise
    raise value.with_traceback(tb)
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03 13:21:04,749 mars.promise 77 ERROR    Exception met in executing promise.
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 301, in <lambda>
    .then(lambda context_dict: _start_calc(context_dict)) \
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 276, in _start_calc
    return self._calc_results(session_id, graph_key, graph, context_dict, chunk_targets)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 383, in _wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 200, in _calc_results
    chunk_targets, retval=False).result()
  File "src/gevent/event.py", line 383, in gevent._gevent_cevent.AsyncResult.result
  File "src/gevent/event.py", line 305, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 335, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 323, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 303, in gevent._gevent_cevent.AsyncResult._raise_exception
  File "/opt/conda/lib/python3.7/site-packages/gevent/_compat.py", line 65, in reraise
    raise value.with_traceback(tb)
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03 13:21:04,751 mars.promise 77 ERROR    Unhandled exception in promise
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 301, in <lambda>
    .then(lambda context_dict: _start_calc(context_dict)) \
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 276, in _start_calc
    return self._calc_results(session_id, graph_key, graph, context_dict, chunk_targets)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 383, in _wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 200, in _calc_results
    chunk_targets, retval=False).result()
  File "src/gevent/event.py", line 383, in gevent._gevent_cevent.AsyncResult.result
  File "src/gevent/event.py", line 305, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 335, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 323, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 303, in gevent._gevent_cevent.AsyncResult._raise_exception
  File "/opt/conda/lib/python3.7/site-packages/gevent/_compat.py", line 65, in reraise
    raise value.with_traceback(tb)
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03 13:21:04,753 mars.promise 77 ERROR    Exception met in executing promise.
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 302, in <lambda>
    .then(lambda keys: _finalize(keys, None), lambda *exc_info: _finalize(None, exc_info))
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 285, in _finalize
    self.tell_promise(callback, *exc_info, _accept=False)
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 530, in tell_promise
    return self.ctx.actor_ref(uid, address=address).tell(callback_args, wait=wait)
  File "mars/actors/core.pyx", line 39, in mars.actors.core.ActorRef.tell
  File "mars/actors/core.pyx", line 41, in mars.actors.core.ActorRef.tell
  File "mars/actors/pool/gevent_pool.pyx", line 282, in mars.actors.pool.gevent_pool.ActorContext.tell
  File "mars/actors/pool/gevent_pool.pyx", line 853, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 859, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 846, in mars.actors.pool.gevent_pool.Communicator._send
  File "mars/actors/pool/gevent_pool.pyx", line 755, in mars.actors.pool.gevent_pool.Communicator._dispatch
  File "mars/actors/pool/gevent_pool.pyx", line 819, in mars.actors.pool.gevent_pool.Communicator._send_process
  File "mars/actors/pool/messages.pyx", line 570, in mars.actors.pool.messages.pack_tell_message
  File "mars/actors/pool/messages.pyx", line 526, in mars.actors.pool.messages._pack_send_message
  File "mars/actors/pool/messages.pyx", line 469, in mars.actors.pool.messages._pack_message
  File "mars/actors/pool/messages.pyx", line 436, in mars.actors.pool.messages._pack_tuple_message
TypeError: can't pickle generator objects
2021-02-03 13:21:04,754 mars.promise 77 ERROR    Unhandled exception in promise
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 302, in <lambda>
    .then(lambda keys: _finalize(keys, None), lambda *exc_info: _finalize(None, exc_info))
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 285, in _finalize
    self.tell_promise(callback, *exc_info, _accept=False)
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 530, in tell_promise
    return self.ctx.actor_ref(uid, address=address).tell(callback_args, wait=wait)
  File "mars/actors/core.pyx", line 39, in mars.actors.core.ActorRef.tell
  File "mars/actors/core.pyx", line 41, in mars.actors.core.ActorRef.tell
  File "mars/actors/pool/gevent_pool.pyx", line 282, in mars.actors.pool.gevent_pool.ActorContext.tell
  File "mars/actors/pool/gevent_pool.pyx", line 853, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 859, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 846, in mars.actors.pool.gevent_pool.Communicator._send
  File "mars/actors/pool/gevent_pool.pyx", line 755, in mars.actors.pool.gevent_pool.Communicator._dispatch
  File "mars/actors/pool/gevent_pool.pyx", line 819, in mars.actors.pool.gevent_pool.Communicator._send_process
2021-02-03 13:21:02,172 mars.worker.execution 58 DEBUG    Executing states: {'402439ba636c37f11bea13fe28ad9bfb': (278.46709537506104, 'CALCULATING'), '4a57865de1482157ed1f3bdd28b8d030': (75.9635705947876, 'CALCULATING')}
  File "mars/actors/pool/messages.pyx", line 570, in mars.actors.pool.messages.pack_tell_message
  File "mars/actors/pool/messages.pyx", line 526, in mars.actors.pool.messages._pack_send_message
  File "mars/actors/pool/messages.pyx", line 469, in mars.actors.pool.messages._pack_message
  File "mars/actors/pool/messages.pyx", line 436, in mars.actors.pool.messages._pack_tuple_message
TypeError: can't pickle generator objects

```

graph 402439ba636c37f11bea13fe28ad9bfb is still calculating in the below log

``` Python
2021-02-03 13:21:32,178 mars.worker.execution 58 DEBUG    Executing states: {'402439ba636c37f11bea13fe28ad9bfb': (308.47222423553467, 'CALCULATING'), '4a57865de1482157ed1f3bdd28b8d030': (105.96869945526123, 'CALCULATING')}
2021-02-03 13:21:42,180 mars.worker.execution 58 DEBUG    Executing states: {'402439ba636c37f11bea13fe28ad9bfb': (318.47462153434753, 'CALCULATING'), '4a57865de1482157ed1f3bdd28b8d030': (115.9710967540741, 'CALCULATING')}

```



Discussion:

Resolving patch:
diff --git a/mars/oscar/__init__.py b/mars/oscar/__init__.py
--- a/mars/oscar/__init__.py
+++ b/mars/oscar/__init__.py
@@ -36,7 +36,13 @@
 from .batch import extensible
 from .core import ActorRef
 from .debug import set_debug_options, get_debug_options, DebugOptions
-from .errors import ActorNotExist, ActorAlreadyExist, ServerClosed, Return
+from .errors import (
+    ActorNotExist,
+    ActorAlreadyExist,
+    ServerClosed,
+    SendMessageFailed,
+    Return,
+)
 from .utils import create_actor_ref
 
 # make sure methods are registered
diff --git a/mars/oscar/backends/pool.py b/mars/oscar/backends/pool.py
--- a/mars/oscar/backends/pool.py
+++ b/mars/oscar/backends/pool.py
@@ -29,7 +29,13 @@
 from ..api import Actor
 from ..core import ActorRef, register_local_pool
 from ..debug import record_message_trace, debug_async_timeout
-from ..errors import ActorAlreadyExist, ActorNotExist, ServerClosed, CannotCancelTask
+from ..errors import (
+    ActorAlreadyExist,
+    ActorNotExist,
+    ServerClosed,
+    CannotCancelTask,
+    SendMessageFailed,
+)
 from ..utils import create_actor_ref
 from .allocate_strategy import allocated_type, AddressSpecified
 from .communication import Channel, Server, get_server_type, gen_local_address
@@ -334,6 +340,33 @@ async def _run_coro(self, message_id: bytes, coro: Coroutine):
         finally:
             self._process_messages.pop(message_id, None)
 
+    async def _send_channel(
+        self, result: _MessageBase, channel: Channel, resend_failure: bool = True
+    ):
+        try:
+            await channel.send(result)
+        except (ChannelClosed, ConnectionResetError):
+            if not self._stopped.is_set():
+                raise
+        except Exception as ex:
+            logger.exception(
+                "Error when sending message %s from %s to %s",
+                result.message_id.hex(),
+                channel.local_address,
+                channel.dest_address,
+            )
+            if not resend_failure:  # pragma: no cover
+                raise
+
+            with _ErrorProcessor(
+                self.external_address, result.message_id, result.protocol
+            ) as processor:
+                raise SendMessageFailed(
+                    f"Error when sending message {result.message_id.hex()}. "
+                    f"Caused by {ex!r}. See server logs for more details"
+                ) from None
+            await self._send_channel(processor.result, channel, resend_failure=False)
+
     async def process_message(self, message: _MessageBase, channel: Channel):
         handler = self._message_handler[message.message_type]
         with _ErrorProcessor(
@@ -349,11 +382,8 @@ async def process_message(self, message: _MessageBase, channel: Channel):
                 processor.result = await self._run_coro(
                     message.message_id, handler(self, message)
                 )
-        try:
-            await channel.send(processor.result)
-        except (ChannelClosed, ConnectionResetError):
-            if not self._stopped.is_set():
-                raise
+
+        await self._send_channel(processor.result, channel)
 
     async def call(self, dest_address: str, message: _MessageBase) -> ResultMessageType:
         return await self._caller.call(self._router, dest_address, message)
diff --git a/mars/oscar/errors.py b/mars/oscar/errors.py
--- a/mars/oscar/errors.py
+++ b/mars/oscar/errors.py
@@ -53,6 +53,10 @@ class CannotCancelTask(MarsError):
     pass
 
 
+class SendMessageFailed(MarsError):
+    pass
+
+
 class Return(MarsError):
     def __init__(self, value):
         self.value = value


Test patch:
diff --git a/mars/core/graph/tests/test_graph.py b/mars/core/graph/tests/test_graph.py
--- a/mars/core/graph/tests/test_graph.py
+++ b/mars/core/graph/tests/test_graph.py
@@ -110,13 +110,15 @@ def test_to_dot():
         graph_reprs = []
         for n in graph:
             graph_reprs.append(
-                f"{n.op.key} -> {[succ.op.key for succ in graph.successors(n)]}"
+                f"{n.op.key} -> {[succ.key for succ in graph.successors(n)]}"
             )
         logging.error(
-            "Unexpected error in test_to_dot.\ndot = %r\ngraph_repr: %r",
+            "Unexpected error in test_to_dot.\ndot = %r\ngraph_repr = %r",
             dot,
             "\n".join(graph_reprs),
         )
-        missing_prefix = next(str(n.key)[5] not in dot for n in graph)
-        logging.error("Missing prefix %s", missing_prefix)
+        missing_prefix = next(n.key for n in graph if str(n.key)[5] not in dot)
+        logging.error(
+            "Missing prefix %r (type: %s)", missing_prefix, type(missing_prefix)
+        )
         raise
diff --git a/mars/oscar/backends/mars/tests/test_pool.py b/mars/oscar/backends/mars/tests/test_pool.py
--- a/mars/oscar/backends/mars/tests/test_pool.py
+++ b/mars/oscar/backends/mars/tests/test_pool.py
@@ -25,7 +25,7 @@
 from .....utils import get_next_port
 from .... import create_actor_ref, Actor, kill_actor
 from ....context import get_context
-from ....errors import NoIdleSlot, ActorNotExist, ServerClosed
+from ....errors import NoIdleSlot, ActorNotExist, ServerClosed, SendMessageFailed
 from ...allocate_strategy import (
     AddressSpecified,
     IdleLabel,
@@ -53,6 +53,11 @@
 from ..pool import MainActorPool, SubActorPool
 
 
+class _CannotBePickled:
+    def __getstate__(self):
+        raise RuntimeError("cannot pickle")
+
+
 class _CannotBeUnpickled:
     def __getstate__(self):
         return ()
@@ -85,6 +90,9 @@ async def sleep(self, second):
     def return_cannot_unpickle(self):
         return _CannotBeUnpickled()
 
+    def raise_cannot_pickle(self):
+        raise ValueError(_CannotBePickled())
+
 
 def _add_pool_conf(
     config: ActorPoolConfig,
@@ -506,6 +514,8 @@ async def test_create_actor_pool():
         assert await actor_ref2.add(1) == 4
         with pytest.raises(RuntimeError):
             await actor_ref2.return_cannot_unpickle()
+        with pytest.raises(SendMessageFailed):
+            await actor_ref2.raise_cannot_pickle()
         assert (await ctx.has_actor(actor_ref2)) is True
         assert (await ctx.actor_ref(actor_ref2)) == actor_ref2
         # test cancel
diff --git a/mars/remote/tests/test_remote_function.py b/mars/remote/tests/test_remote_function.py
--- a/mars/remote/tests/test_remote_function.py
+++ b/mars/remote/tests/test_remote_function.py
@@ -20,6 +20,7 @@
 
 from ... import dataframe as md
 from ... import tensor as mt
+from ... import oscar as mo
 from ...core import tile
 from ...deploy.oscar.session import get_default_session
 from ...learn.utils import shuffle
@@ -188,3 +189,16 @@ def f(*_args):
     r4 = spawn(f, args=(r2, r3))
 
     assert r4.execute().fetch() is None
+
+
+def test_remote_with_unpickable(setup_cluster):
+    def f(*_):
+        class Unpickleable:
+            def __reduce__(self):
+                raise ValueError
+
+        raise KeyError(Unpickleable())
+
+    with pytest.raises(mo.SendMessageFailed):
+        d = spawn(f, retry_when_fail=False)
+        d.execute()