Repository: mars-project/mars

Issue:
[BUG] Operand runs forever when error message can't be pickled
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Operand raised an error which can't be pickled, the status of the operand is still `CALCULATING` and never convert to `FAILED`.

``` Python
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03T05:21:04Z (<ThreadPoolWorker at 0x7f14c2e8e7e0 thread_ident=0x7f14cfe3f700 threadpool-hub=<Hub at 0x7f14c338d1d0 thread_ident=0x7f14f8070740>>, <cyfunction GeventThreadPool._wrap_watch.<locals>.inner at 0x7f14c2ec37a0>) failed with ConnectionError

2021-02-03 13:21:04,748 mars.worker.calc 77 ERROR    Unexpected exception occurred in BaseCalcActor._calc_results. graph_key=402439ba636c37f11bea13fe28ad9bfb
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 383, in _wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 200, in _calc_results
    chunk_targets, retval=False).result()
  File "src/gevent/event.py", line 383, in gevent._gevent_cevent.AsyncResult.result
  File "src/gevent/event.py", line 305, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 335, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 323, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 303, in gevent._gevent_cevent.AsyncResult._raise_exception
  File "/opt/conda/lib/python3.7/site-packages/gevent/_compat.py", line 65, in reraise
    raise value.with_traceback(tb)
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03 13:21:04,749 mars.promise 77 ERROR    Exception met in executing promise.
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 301, in <lambda>
    .then(lambda context_dict: _start_calc(context_dict)) \
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 276, in _start_calc
    return self._calc_results(session_id, graph_key, graph, context_dict, chunk_targets)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 383, in _wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 200, in _calc_results
    chunk_targets, retval=False).result()
  File "src/gevent/event.py", line 383, in gevent._gevent_cevent.AsyncResult.result
  File "src/gevent/event.py", line 305, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 335, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 323, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 303, in gevent._gevent_cevent.AsyncResult._raise_exception
  File "/opt/conda/lib/python3.7/site-packages/gevent/_compat.py", line 65, in reraise
    raise value.with_traceback(tb)
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03 13:21:04,751 mars.promise 77 ERROR    Unhandled exception in promise
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 479, in send
    r = low_conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 482, in send
    r = low_conn.getresponse()
  File "/opt/conda/lib/python3.7/http/client.py", line 1344, in getresponse
    response.begin()
  File "/opt/conda/lib/python3.7/http/client.py", line 306, in begin
    version, status, reason = self._read_status()
  File "/opt/conda/lib/python3.7/http/client.py", line 267, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/opt/conda/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 301, in <lambda>
    .then(lambda context_dict: _start_calc(context_dict)) \
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 276, in _start_calc
    return self._calc_results(session_id, graph_key, graph, context_dict, chunk_targets)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 383, in _wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 200, in _calc_results
    chunk_targets, retval=False).result()
  File "src/gevent/event.py", line 383, in gevent._gevent_cevent.AsyncResult.result
  File "src/gevent/event.py", line 305, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 335, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 323, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 303, in gevent._gevent_cevent.AsyncResult._raise_exception
  File "/opt/conda/lib/python3.7/site-packages/gevent/_compat.py", line 65, in reraise
    raise value.with_traceback(tb)
  File "/opt/conda/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 201, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 698, in execute_graph
    res = graph_execution.execute(retval)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 579, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/utils.py", line 457, in _inner
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 446, in _execute_operand
    self.handle_op(first_op, results, self._mock)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 378, in handle_op
    return Executor.handle(*args, **kw)
  File "/opt/conda/lib/python3.7/site-packages/mars/executor.py", line 649, in handle
    return runner(results, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 359, in execute
    cls._execute_arrow_tunnel(ctx, op)
  File "/opt/conda/lib/python3.7/site-packages/odps/mars_extension/dataframe/datastore.py", line 349, in _execute_arrow_tunnel
    writer.close()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 548, in close
    self._flush()
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/io/writer.py", line 545, in _flush
    self._request_callback(gen())
  File "/opt/conda/lib/python3.7/site-packages/odps/tunnel/tabletunnel.py", line 315, in upload
    self._client.put(url, data=data, params=params, headers=headers)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 154, in put
    return self.request(url, 'put', data=data, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/odps/rest.py", line 133, in request
    proxies=self._proxy)
  File "/opt/conda/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: timed out
2021-02-03 13:21:04,753 mars.promise 77 ERROR    Exception met in executing promise.
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 302, in <lambda>
    .then(lambda keys: _finalize(keys, None), lambda *exc_info: _finalize(None, exc_info))
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 285, in _finalize
    self.tell_promise(callback, *exc_info, _accept=False)
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 530, in tell_promise
    return self.ctx.actor_ref(uid, address=address).tell(callback_args, wait=wait)
  File "mars/actors/core.pyx", line 39, in mars.actors.core.ActorRef.tell
  File "mars/actors/core.pyx", line 41, in mars.actors.core.ActorRef.tell
  File "mars/actors/pool/gevent_pool.pyx", line 282, in mars.actors.pool.gevent_pool.ActorContext.tell
  File "mars/actors/pool/gevent_pool.pyx", line 853, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 859, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 846, in mars.actors.pool.gevent_pool.Communicator._send
  File "mars/actors/pool/gevent_pool.pyx", line 755, in mars.actors.pool.gevent_pool.Communicator._dispatch
  File "mars/actors/pool/gevent_pool.pyx", line 819, in mars.actors.pool.gevent_pool.Communicator._send_process
  File "mars/actors/pool/messages.pyx", line 570, in mars.actors.pool.messages.pack_tell_message
  File "mars/actors/pool/messages.pyx", line 526, in mars.actors.pool.messages._pack_send_message
  File "mars/actors/pool/messages.pyx", line 469, in mars.actors.pool.messages._pack_message
  File "mars/actors/pool/messages.pyx", line 436, in mars.actors.pool.messages._pack_tuple_message
TypeError: can't pickle generator objects
2021-02-03 13:21:04,754 mars.promise 77 ERROR    Unhandled exception in promise
Traceback (most recent call last):
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 302, in <lambda>
    .then(lambda keys: _finalize(keys, None), lambda *exc_info: _finalize(None, exc_info))
  File "/opt/conda/lib/python3.7/site-packages/mars/worker/calc.py", line 285, in _finalize
    self.tell_promise(callback, *exc_info, _accept=False)
  File "/opt/conda/lib/python3.7/site-packages/mars/promise.py", line 530, in tell_promise
    return self.ctx.actor_ref(uid, address=address).tell(callback_args, wait=wait)
  File "mars/actors/core.pyx", line 39, in mars.actors.core.ActorRef.tell
  File "mars/actors/core.pyx", line 41, in mars.actors.core.ActorRef.tell
  File "mars/actors/pool/gevent_pool.pyx", line 282, in mars.actors.pool.gevent_pool.ActorContext.tell
  File "mars/actors/pool/gevent_pool.pyx", line 853, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 859, in mars.actors.pool.gevent_pool.Communicator.tell
  File "mars/actors/pool/gevent_pool.pyx", line 846, in mars.actors.pool.gevent_pool.Communicator._send
  File "mars/actors/pool/gevent_pool.pyx", line 755, in mars.actors.pool.gevent_pool.Communicator._dispatch
  File "mars/actors/pool/gevent_pool.pyx", line 819, in mars.actors.pool.gevent_pool.Communicator._send_process
2021-02-03 13:21:02,172 mars.worker.execution 58 DEBUG    Executing states: {'402439ba636c37f11bea13fe28ad9bfb': (278.46709537506104, 'CALCULATING'), '4a57865de1482157ed1f3bdd28b8d030': (75.9635705947876, 'CALCULATING')}
  File "mars/actors/pool/messages.pyx", line 570, in mars.actors.pool.messages.pack_tell_message
  File "mars/actors/pool/messages.pyx", line 526, in mars.actors.pool.messages._pack_send_message
  File "mars/actors/pool/messages.pyx", line 469, in mars.actors.pool.messages._pack_message
  File "mars/actors/pool/messages.pyx", line 436, in mars.actors.pool.messages._pack_tuple_message
TypeError: can't pickle generator objects

```

graph 402439ba636c37f11bea13fe28ad9bfb is still calculating in the below log

``` Python
2021-02-03 13:21:32,178 mars.worker.execution 58 DEBUG    Executing states: {'402439ba636c37f11bea13fe28ad9bfb': (308.47222423553467, 'CALCULATING'), '4a57865de1482157ed1f3bdd28b8d030': (105.96869945526123, 'CALCULATING')}
2021-02-03 13:21:42,180 mars.worker.execution 58 DEBUG    Executing states: {'402439ba636c37f11bea13fe28ad9bfb': (318.47462153434753, 'CALCULATING'), '4a57865de1482157ed1f3bdd28b8d030': (115.9710967540741, 'CALCULATING')}

```



Discussion:

Resolving patch:
diff --git a/mars/tensor/statistics/histogram.py b/mars/tensor/statistics/histogram.py
--- a/mars/tensor/statistics/histogram.py
+++ b/mars/tensor/statistics/histogram.py
@@ -505,11 +505,11 @@ def __call__(self, a, bins, range, weights):
             inputs.append(weights)
         if (a.size > 0 or np.isnan(a.size)) and \
                 (isinstance(bins, str) or mt.ndim(bins) == 0) and not range:
-            # for bins that is str or integer,
-            # requires min max calculated first
-            input_min = self._input_min = a.min()
+            # for bins that is str or integer, requires min max calculated first
+            # dims need to be kept in case a is empty which causes errors in reduction
+            input_min = self._input_min = a.min(keepdims=True)
             inputs.append(input_min)
-            input_max = self._input_max = a.max()
+            input_max = self._input_max = a.max(keepdims=True)
             inputs.append(input_max)
 
         return self.new_tensor(inputs, shape=shape, order=TensorOrder.C_ORDER)
@@ -527,7 +527,10 @@ def tile(cls, op):
             metas = ctx.get_chunk_metas(min_max_chunk_keys)
             if any(meta is None for meta in metas):
                 raise TilesError('`input_min` or `input_max` need be executed first')
-            range_ = tuple(ctx.get_chunk_results(min_max_chunk_keys))
+            range_results = ctx.get_chunk_results(min_max_chunk_keys)
+            # make sure returned bounds are valid
+            if all(x.size > 0 for x in range_results):
+                range_ = tuple(x[0] for x in range_results)
         if isinstance(op.bins, TENSOR_TYPE):
             # `bins` is a Tensor, needs to be calculated first
             bins_chunk_keys = [c.key for c in op.bins.chunks]
diff --git a/mars/worker/calc.py b/mars/worker/calc.py
--- a/mars/worker/calc.py
+++ b/mars/worker/calc.py
@@ -21,7 +21,7 @@
 from ..executor import Executor
 from ..serialize import dataserializer
 from ..utils import to_str, deserialize_graph, log_unhandled, calc_data_size, \
-    get_chunk_shuffle_key
+    get_chunk_shuffle_key, build_exc_info
 from ..context import DistributedDictContext
 from .events import EventContext, EventCategory, EventLevel, ProcedureEventType
 from .storage import DataStorageDevice
@@ -282,7 +282,13 @@ def _finalize(keys, exc_info):
             if not exc_info:
                 self.tell_promise(callback, keys)
             else:
-                self.tell_promise(callback, *exc_info, _accept=False)
+                try:
+                    self.tell_promise(callback, *exc_info, _accept=False)
+                except:
+                    self.tell_promise(callback, *build_exc_info(
+                        SystemError, f'Failed to send errors to scheduler, type: {exc_info[0].__name__}, '
+                                     f'message: {str(exc_info[1])}'), _accept=False)
+                    raise
 
             keys_to_release = [k for k in keys_to_fetch if get_chunk_key(k) not in chunk_targets]
             if exc_info:


Test patch:
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -1237,6 +1237,20 @@ def f5(t, x):
             expected = (raw[raw < 0.5] * 3).sum()
             self.assertAlmostEqual(result, expected, places=5)
 
+            # test execution with unpickable errors
+            session6 = new_session(cluster.endpoint)
+
+            def f6():
+                class Unpickleable:
+                    def __reduce__(self):
+                        raise ValueError
+
+                raise KeyError(Unpickleable())
+
+            d = mr.spawn(f6, retry_when_fail=False)
+            with self.assertRaises(ExecutionFailed):
+                session6.run(d, timeout=_exec_timeout)
+
     @unittest.skipIf(sklearn is None, 'sklearn not installed')
     def testLearnInLocalCluster(self, *_):
         from mars.learn.cluster import KMeans