Repository: mars-project/mars

Issue:
[BUG] Failed to execute tiled tensor in distributed mode
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Failed to execute a tiled tensor in local cluster mode, the exception occurred when fetching the tensor result.

The issue may relate to this line: https://github.com/mars-project/mars/blob/master/mars/scheduler/graph.py#L393
the tiled tensor is not coarse, it is skipped and not recorded by `_tensor_key_opid_to_tiled `, `IndexError` is raised when build merge graph.

**To Reproduce**
``` Python
In [1]: from mars.deploy.local import new_cluster                                                           

In [2]: cluster = new_cluster(scheduler_n_process=2, worker_n_process=3, shared_memory='100M')              
/Users/travis/build/wesm/crossbow/arrow/cpp/src/plasma/store.cc:971: Allowing the Plasma store to use up to 0.104858GB of memory.
/Users/travis/build/wesm/crossbow/arrow/cpp/src/plasma/store.cc:1001: Starting object store with directory /tmp and huge page support disabled

In [3]: import mars.tensor as mt                                                                            

In [4]: a = mt.random.rand(10, 10, chunk_size=3)                                                            

In [5]: a.tiles()                                                                                           
Out[5]: Tensor <op=TensorRand, shape=(10, 10), key=2d1101e0bd65a1ca3134ca872c6072b5>

In [6]: cluster.session.run(a)                                                                              
Unexpected exception occurred in GraphActor.build_tensor_merge_graph. tensor_key='2d1101e0bd65a1ca3134ca872c6072b5'
Traceback (most recent call last):
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/utils.py", line 298, in _wrapped
    return func(*args, **kwargs)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 819, in build_tensor_merge_graph
    tiled_tensor = self._get_tensor_by_key(tensor_key)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 804, in _get_tensor_by_key
    return self._tensor_key_opid_to_tiled[(key, tid)][-1]
IndexError: list index out of range
Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
  File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/utils.py", line 298, in _wrapped
    return func(*args, **kwargs)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 819, in build_tensor_merge_graph
    tiled_tensor = self._get_tensor_by_key(tensor_key)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 804, in _get_tensor_by_key
    return self._tensor_key_opid_to_tiled[(key, tid)][-1]
IndexError: list index out of range
2019-03-18T03:29:21Z <Greenlet at 0xc17966268: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0xc1796f818>> failed with IndexError

Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
  File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 66, in fetch_tensor
    fetch_graph = deserialize_graph(graph_actor.build_tensor_merge_graph(tensor_key))
  File "mars/actors/core.pyx", line 64, in mars.actors.core.ActorRef.__getattr__._mt_call
  File "mars/actors/core.pyx", line 36, in mars.actors.core.ActorRef.send
  File "mars/actors/pool/gevent_pool.pyx", line 179, in mars.actors.pool.gevent_pool.ActorContext.send
  File "mars/actors/pool/gevent_pool.pyx", line 754, in mars.actors.pool.gevent_pool.Communicator.send
  File "mars/actors/pool/gevent_pool.pyx", line 755, in mars.actors.pool.gevent_pool.Communicator.send
  File "mars/actors/pool/gevent_pool.pyx", line 750, in mars.actors.pool.gevent_pool.Communicator._send
  File "mars/actors/pool/gevent_pool.pyx", line 659, in mars.actors.pool.gevent_pool.Communicator._dispatch
  File "mars/actors/pool/gevent_pool.pyx", line 730, in mars.actors.pool.gevent_pool.Communicator._send_process
  File "mars/actors/pool/gevent_pool.pyx", line 278, in mars.actors.pool.gevent_pool.AsyncHandler.submit
  File "src/gevent/event.py", line 344, in gevent._event.AsyncResult.result
  File "src/gevent/event.py", line 268, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 286, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 266, in gevent._event.AsyncResult._raise_exception
  File "/Users/hekaisheng/miniconda3/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
    raise value.with_traceback(tb)
  File "mars/actors/pool/gevent_pool.pyx", line 942, in mars.actors.pool.gevent_pool.Communicator._on_receive_send
  File "mars/actors/pool/gevent_pool.pyx", line 73, in mars.actors.pool.gevent_pool.MessageContext.result
  File "src/gevent/event.py", line 344, in gevent._event.AsyncResult.result
  File "src/gevent/event.py", line 268, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 296, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 286, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 266, in gevent._event.AsyncResult._raise_exception
  File "/Users/hekaisheng/miniconda3/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
    raise value.with_traceback(tb)
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/utils.py", line 298, in _wrapped
    return func(*args, **kwargs)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 819, in build_tensor_merge_graph
    tiled_tensor = self._get_tensor_by_key(tensor_key)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 804, in _get_tensor_by_key
    return self._tensor_key_opid_to_tiled[(key, tid)][-1]
IndexError: list index out of range
2019-03-18T03:29:21Z <Greenlet at 0xc179668c8: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0xc17968408>> failed with IndexError

Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
  File "mars/actors/pool/gevent_pool.pyx", line 446, in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote
  File "mars/actors/pool/gevent_pool.pyx", line 450, in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote
  File "mars/actors/pool/gevent_pool.pyx", line 457, in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/lib/six.py", line 702, in reraise
    raise value.with_traceback(tb)
  File "mars/actors/pool/gevent_pool.pyx", line 942, in mars.actors.pool.gevent_pool.Communicator._on_receive_send
  File "mars/actors/pool/gevent_pool.pyx", line 73, in mars.actors.pool.gevent_pool.MessageContext.result
  File "src/gevent/event.py", line 344, in gevent._event.AsyncResult.result
  File "src/gevent/event.py", line 268, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 296, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 286, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 266, in gevent._event.AsyncResult._raise_exception
  File "/Users/hekaisheng/miniconda3/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
    raise value.with_traceback(tb)
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
    cpdef on_receive(self, message):
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
    return getattr(self, method)(*args, **kwargs)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 66, in fetch_tensor
    fetch_graph = deserialize_graph(graph_actor.build_tensor_merge_graph(tensor_key))
  File "mars/actors/core.pyx", line 64, in mars.actors.core.ActorRef.__getattr__._mt_call
    return self.send((item,) + args + (kwargs,), wait=wait)
  File "mars/actors/core.pyx", line 36, in mars.actors.core.ActorRef.send
    return self._ctx.send(self, message, wait=wait, callback=callback)
  File "mars/actors/pool/gevent_pool.pyx", line 179, in mars.actors.pool.gevent_pool.ActorContext.send
  File "mars/actors/pool/gevent_pool.pyx", line 754, in mars.actors.pool.gevent_pool.Communicator.send
  File "mars/actors/pool/gevent_pool.pyx", line 755, in mars.actors.pool.gevent_pool.Communicator.send
  File "mars/actors/pool/gevent_pool.pyx", line 750, in mars.actors.pool.gevent_pool.Communicator._send
  File "mars/actors/pool/gevent_pool.pyx", line 659, in mars.actors.pool.gevent_pool.Communicator._dispatch
  File "mars/actors/pool/gevent_pool.pyx", line 730, in mars.actors.pool.gevent_pool.Communicator._send_process
  File "mars/actors/pool/gevent_pool.pyx", line 278, in mars.actors.pool.gevent_pool.AsyncHandler.submit
  File "src/gevent/event.py", line 344, in gevent._event.AsyncResult.result
  File "src/gevent/event.py", line 268, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 286, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 266, in gevent._event.AsyncResult._raise_exception
  File "/Users/hekaisheng/miniconda3/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
    raise value.with_traceback(tb)
  File "mars/actors/pool/gevent_pool.pyx", line 942, in mars.actors.pool.gevent_pool.Communicator._on_receive_send
  File "mars/actors/pool/gevent_pool.pyx", line 73, in mars.actors.pool.gevent_pool.MessageContext.result
  File "src/gevent/event.py", line 344, in gevent._event.AsyncResult.result
  File "src/gevent/event.py", line 268, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 296, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 286, in gevent._event.AsyncResult.get
  File "src/gevent/event.py", line 266, in gevent._event.AsyncResult._raise_exception
  File "/Users/hekaisheng/miniconda3/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
    raise value.with_traceback(tb)
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
    cpdef on_receive(self, message):
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
    return getattr(self, method)(*args, **kwargs)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/utils.py", line 298, in _wrapped
    return func(*args, **kwargs)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 819, in build_tensor_merge_graph
    tiled_tensor = self._get_tensor_by_key(tensor_key)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 804, in _get_tensor_by_key
    return self._tensor_key_opid_to_tiled[(key, tid)][-1]
IndexError: list index out of range
2019-03-18T03:29:21Z <Greenlet at 0xc17962378: <built-in method _send_remote of mars.actors.pool.gevent_pool.ActorRemoteHelper object at 0xc17860e58>('0.0.0.0:29790', [bytearray(b'\x05\x01 \x00\x00\x00\x00\x00\x00\x00)> failed with IndexError

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-6-06ec9ebba039> in <module>
----> 1 cluster.session.run(a)

~/Documents/mars_dev/mars/mars/session.py in run(self, *tensors, **kw)
    107 
    108         tensors = tuple(mt.tensor(t) for t in tensors)
--> 109         result = self._sess.run(*tensors, **kw)
    110 
    111         for t in tensors:

~/Documents/mars_dev/mars/mars/deploy/local/session.py in run(self, *tensors, **kw)
    103             return
    104         else:
--> 105             return self.fetch(*tensors)
    106 
    107     def fetch(self, *tensors):

~/Documents/mars_dev/mars/mars/deploy/local/session.py in fetch(self, *tensors)
    116             future = self._api.fetch_data(self._session_id, graph_key, key, wait=False)
    117             futures.append(future)
--> 118         return [dataserializer.loads(f.result()) for f in futures]
    119 
    120     def decref(self, *keys):

~/Documents/mars_dev/mars/mars/deploy/local/session.py in <listcomp>(.0)
    116             future = self._api.fetch_data(self._session_id, graph_key, key, wait=False)
    117             futures.append(future)
--> 118         return [dataserializer.loads(f.result()) for f in futures]
    119 
    120     def decref(self, *keys):

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.result()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult._raise_exception()

~/miniconda3/lib/python3.7/site-packages/gevent/_compat.py in reraise(t, value, tb)
     45     def reraise(t, value, tb=None): # pylint:disable=unused-argument
     46         if value.__traceback__ is not tb and tb is not None:
---> 47             raise value.with_traceback(tb)
     48         raise value
     49     def exc_clear():

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._async_run.on_failure()

~/miniconda3/lib/python3.7/site-packages/gevent/_greenlet.cpython-37m-darwin.so in gevent._greenlet.Greenlet.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_greenlet.cpython-37m-darwin.so in gevent._greenlet.Greenlet._raise_exception()

~/miniconda3/lib/python3.7/site-packages/gevent/_compat.py in reraise()
     45     def reraise(t, value, tb=None): # pylint:disable=unused-argument
     46         if value.__traceback__ is not tb and tb is not None:
---> 47             raise value.with_traceback(tb)
     48         raise value
     49     def exc_clear():

~/miniconda3/lib/python3.7/site-packages/gevent/_greenlet.cpython-37m-darwin.so in gevent._greenlet.Greenlet.run()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote()

~/Documents/mars_dev/mars/mars/lib/six.py in reraise()
    700                 value = tp()
    701             if value.__traceback__ is not tb:
--> 702                 raise value.with_traceback(tb)
    703             raise value
    704         finally:

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.Communicator._on_receive_send()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.MessageContext.result()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.result()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult._raise_exception()

~/miniconda3/lib/python3.7/site-packages/gevent/_compat.py in reraise()
     45     def reraise(t, value, tb=None): # pylint:disable=unused-argument
     46         if value.__traceback__ is not tb and tb is not None:
---> 47             raise value.with_traceback(tb)
     48         raise value
     49     def exc_clear():

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run()

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.FunctionActor.on_receive()
    106 
    107 cdef class FunctionActor(Actor):
--> 108     cpdef on_receive(self, message):
    109         method, args, kwargs = message[0], message[1:-1], message[-1]
    110         return getattr(self, method)(*args, **kwargs)

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.FunctionActor.on_receive()
    108     cpdef on_receive(self, message):
    109         method, args, kwargs = message[0], message[1:-1], message[-1]
--> 110         return getattr(self, method)(*args, **kwargs)
    111 
    112 

~/Documents/mars_dev/mars/mars/scheduler/graph.py in fetch_tensor()
     64 
     65         graph_actor = self.ctx.actor_ref(GraphActor.gen_name(session_id, graph_key))
---> 66         fetch_graph = deserialize_graph(graph_actor.build_tensor_merge_graph(tensor_key))
     67 
     68         if len(fetch_graph) == 1 and isinstance(next(fetch_graph.iter_nodes()).op, TensorFetch):

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.ActorRef.__getattr__._mt_call()
     62                 return self.tell((item,) + args + (kwargs,), delay=delay, wait=wait)
     63             else:
---> 64                 return self.send((item,) + args + (kwargs,), wait=wait)
     65 
     66         return _mt_call

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.ActorRef.send()
     34 
     35     cpdef object send(self, object message, bint wait=True, object callback=None):
---> 36         return self._ctx.send(self, message, wait=wait, callback=callback)
     37 
     38     cpdef object tell(self, object message, object delay=None, bint wait=True,

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorContext.send()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.Communicator.send()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.Communicator.send()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.Communicator._send()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.Communicator._dispatch()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.Communicator._send_process()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.AsyncHandler.submit()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.result()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult._raise_exception()

~/miniconda3/lib/python3.7/site-packages/gevent/_compat.py in reraise()
     45     def reraise(t, value, tb=None): # pylint:disable=unused-argument
     46         if value.__traceback__ is not tb and tb is not None:
---> 47             raise value.with_traceback(tb)
     48         raise value
     49     def exc_clear():

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.Communicator._on_receive_send()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.MessageContext.result()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.result()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult.get()

~/miniconda3/lib/python3.7/site-packages/gevent/_event.cpython-37m-darwin.so in gevent._event.AsyncResult._raise_exception()

~/miniconda3/lib/python3.7/site-packages/gevent/_compat.py in reraise()
     45     def reraise(t, value, tb=None): # pylint:disable=unused-argument
     46         if value.__traceback__ is not tb and tb is not None:
---> 47             raise value.with_traceback(tb)
     48         raise value
     49     def exc_clear():

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run()

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.FunctionActor.on_receive()
    106 
    107 cdef class FunctionActor(Actor):
--> 108     cpdef on_receive(self, message):
    109         method, args, kwargs = message[0], message[1:-1], message[-1]
    110         return getattr(self, method)(*args, **kwargs)

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.FunctionActor.on_receive()
    108     cpdef on_receive(self, message):
    109         method, args, kwargs = message[0], message[1:-1], message[-1]
--> 110         return getattr(self, method)(*args, **kwargs)
    111 
    112 

~/Documents/mars_dev/mars/mars/utils.py in _wrapped()
    296     def _wrapped(*args, **kwargs):
    297         try:
--> 298             return func(*args, **kwargs)
    299         except:  # noqa: E722
    300             kwcopy = kwargs.copy()

~/Documents/mars_dev/mars/mars/scheduler/graph.py in build_tensor_merge_graph()
    817         from ..tensor.expressions.merge.concatenate import TensorConcatenate
    818 
--> 819         tiled_tensor = self._get_tensor_by_key(tensor_key)
    820         graph = DAG()
    821         if len(tiled_tensor.chunks) == 1:

~/Documents/mars_dev/mars/mars/scheduler/graph.py in _get_tensor_by_key()
    802     def _get_tensor_by_key(self, key):
    803         tid = self._tensor_key_to_opid[key]
--> 804         return self._tensor_key_opid_to_tiled[(key, tid)][-1]
    805 
    806     @log_unhandled

IndexError: list index out of range

```



Discussion:

Resolving patch:
diff --git a/mars/core.py b/mars/core.py
--- a/mars/core.py
+++ b/mars/core.py
@@ -66,14 +66,14 @@ def __init__(self, *args, **kwargs):
         super(BaseWithKey, self).__init__(*args, **kwargs)
 
         if self._init_update_key_ and (not hasattr(self, '_key') or not self._key):
-            self.update_key()
+            self._update_key()
         if not hasattr(self, '_id') or not self._id:
             self._id = str(id(self))
 
     def _obj_set(self, k, v):
         object.__setattr__(self, k, v)
 
-    def update_key(self):
+    def _update_key(self):
         self._obj_set('_key', tokenize(type(self), *self._values_))
         return self
 
@@ -81,10 +81,6 @@ def reset_key(self):
         self._obj_set('_key', None)
         return self
 
-    def update_id(self, new_id=None):
-        new_id = new_id if new_id is not None else str(id(self))
-        self._obj_set('_id', new_id)
-
     def __copy__(self):
         return self.copy()
 
@@ -279,7 +275,7 @@ def is_sparse(self):
 
     issparse = is_sparse
 
-    def update_key(self):
+    def _update_key(self):
         object.__setattr__(self, '_key', tokenize(
             type(self), *(getattr(self, k, None) for k in self._keys_ if k != '_index')))
 
@@ -388,6 +384,16 @@ def cix(self):
     def is_coarse(self):
         return not hasattr(self, '_chunks') or self._chunks is None or len(self._chunks) == 0
 
+    def to_coarse(self):
+        if self.is_coarse():
+            return self
+        new_entity = self.copy()
+        new_entity._obj_set('_id', self._id)
+        new_entity._chunks = None
+        if self.inputs is None or len(self.inputs) == 0:
+            new_entity.params.update({'raw_chunk_size': self.nsplits})
+        return new_entity
+
     def tiles(self):
         return handler.tiles(self)
 
@@ -409,6 +415,7 @@ def build_graph(self, graph=None, cls=DAG, tiled=False, compose=True, executed_k
             keys = list(c.key for c in self.chunks)
         else:
             nodes = list(self.op.outputs)
+
         visited = set()
         while len(nodes) > 0:
             node = nodes.pop()
@@ -432,8 +439,29 @@ def build_graph(self, graph=None, cls=DAG, tiled=False, compose=True, executed_k
                           if c not in visited])
         if tiled and compose:
             graph.compose(keys=keys)
+
+        if not tiled and any(not n.is_coarse() for n in graph):
+            return self._to_coarse_graph(graph)
+
         return graph
 
+    @staticmethod
+    def _to_coarse_graph(graph):
+        new_graph = type(graph)()
+        visited = dict()
+        for n in graph:
+            if n not in visited:
+                new_node = n.to_coarse()
+                visited[n] = new_node
+                new_graph.add_node(new_node)
+            for succ in graph.successors(n):
+                if succ not in visited:
+                    new_node = succ.to_coarse()
+                    visited[succ] = new_node
+                    new_graph.add_node(new_node)
+                new_graph.add_edge(visited[n], visited[succ])
+        return new_graph
+
     def visualize(self, graph_attrs=None, node_attrs=None, **kw):
         from graphviz import Source
 
@@ -482,7 +510,7 @@ def _new_chunks(self, inputs, shape, index=None, output_limit=None, kws=None, **
         self.check_inputs(inputs)
         getattr(self, '_set_inputs')(inputs)
         if getattr(self, '_key', None) is None:
-            getattr(self, 'update_key')()  # update key when inputs are set
+            getattr(self, '_update_key')()  # update key when inputs are set
 
         if isinstance(shape, (list, tuple)) and len(shape) > 0 and isinstance(shape[0], (list, tuple)):
             if len(shape) != output_limit:
@@ -531,7 +559,7 @@ def _new_entities(self, inputs, shape, chunks=None, nsplits=None, output_limit=N
         self.check_inputs(inputs)
         getattr(self, '_set_inputs')(inputs)
         if getattr(self, '_key', None) is None:
-            getattr(self, 'update_key')()  # update key when inputs are set
+            getattr(self, '_update_key')()  # update key when inputs are set
 
         if isinstance(shape, (list, tuple)) and len(shape) > 0 and isinstance(shape[0], (list, tuple)):
             if not np.isinf(output_limit) and len(shape) != output_limit:
diff --git a/mars/operands/random.py b/mars/operands/random.py
--- a/mars/operands/random.py
+++ b/mars/operands/random.py
@@ -44,7 +44,7 @@ def args(self):
         return [slot for slot in self.__slots__
                 if slot not in set(RandomOperand.__slots__)]
 
-    def update_key(self):
+    def _update_key(self):
         args = tuple(getattr(self, k, None) for k in self._keys_)
         if self.state is None:
             args += (np.random.random(),)


Test patch:
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -491,3 +491,20 @@ def testExistingOperand(self):
             d = d.dot(d)
             r4 = session.run(d, compose=False)
             np.testing.assert_array_equal(r4, np.ones((5, 5)) * 5)
+
+    def testTiledTensor(self):
+        with new_cluster(scheduler_n_process=2, worker_n_process=2,
+                         shared_memory='20M') as cluster:
+            session = cluster.session
+            a = mt.ones((10, 10), chunk_size=3)
+            b = a.dot(a)
+            b.tiles()
+
+            r = session.run(b)
+            np.testing.assert_array_equal(r, np.ones((10, 10)) * 10)
+
+            a.tiles()
+            b = a + 1
+
+            r = session.run(b)
+            np.testing.assert_array_equal(r, np.ones((10, 10)) + 1)