Repository: mars-project/mars

Issue:
[BUG] got an IndexError when setitem with mt.ones
**Describe the bug**
When submit setitem expressions to cluster, `GraphActor.prepare_graph` raised `IndexError`.

**To Reproduce**
```
In [4]: a = mt.random.rand(10, 10)

In [5]: a[:5, :5] = mt.ones((5, 5))

In [6]: a[5:10, 5:10] = mt.ones((5, 5)) * 2

In [7]: cluster.session.run(a)
Failed to start graph execution.
Traceback (most recent call last):
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 239, in execute_graph
    self.prepare_graph()
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 416, in prepare_graph
    for n in tensor_to_tiled[tk][-1].chunks:
IndexError: list index out of range
Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 716, in gevent._greenlet.Greenlet.run
  File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 239, in execute_graph
    self.prepare_graph()
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 416, in prepare_graph
    for n in tensor_to_tiled[tk][-1].chunks:
IndexError: list index out of range
2019-01-08T06:55:42Z <Greenlet "Greenlet-0" at 0x320317c48: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0x32047c458>> failed with IndexError

---------------------------------------------------------------------------
SystemError                               Traceback (most recent call last)
<ipython-input-7-06ec9ebba039> in <module>()
----> 1 cluster.session.run(a)

~/Documents/mars_dev/mars/mars/session.py in run(self, *tensors, **kw)
    110         # execute the non-executed tensors
    111         if idx_to_run_tensors:
--> 112             execute_result = self._sess.run(*idx_to_run_tensors.values(), **kw)
    113             if execute_result:
    114                 # fetch is True

~/Documents/mars_dev/mars/mars/deploy/local/session.py in run(self, *tensors, **kw)
     71             if graph_state == GraphState.FAILED:
     72                 # TODO(qin): add traceback
---> 73                 raise SystemError('Graph execution failed with unknown reason')
     74 
     75         if 0 < timeout < time.time() - exec_start_time:

SystemError: Graph execution failed with unknown reason
```
however it works when value is `mt.ones((5, 5))` without multiply by 2.
```
In [4]: a = mt.random.rand(10, 10)

In [5]: a[:5, :5] = mt.ones((5, 5))

In [6]: a[5:10, 5:10] = mt.ones((5, 5))

In [7]: cluster.session.run(a)
Out[7]: 
array([[1.        , 1.        , 1.        , 1.        , 1.        ,
        0.71645692, 0.73310965, 0.27332111, 0.53404958, 0.43818169],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.7654411 , 0.06258097, 0.32313892, 0.67699528, 0.83672464],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.38562723, 0.51791558, 0.92523203, 0.88885575, 0.88152387],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.85278501, 0.15031892, 0.36973803, 0.79156834, 0.25785725],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.01598595, 0.35417422, 0.51862623, 0.44287278, 0.79882332],
       [0.98480177, 0.88607345, 0.78764026, 0.48536084, 0.59666905,
        1.        , 1.        , 1.        , 1.        , 1.        ],
       [0.63707366, 0.88748085, 0.85763743, 0.64301722, 0.58708402,
        1.        , 1.        , 1.        , 1.        , 1.        ],
       [0.91564735, 0.57704163, 0.41733464, 0.17699732, 0.17902524,
        1.        , 1.        , 1.        , 1.        , 1.        ],
       [0.6412424 , 0.56966859, 0.98603185, 0.29877888, 0.64962717,
        1.        , 1.        , 1.        , 1.        , 1.        ],
       [0.22425854, 0.04535534, 0.4892396 , 0.78086597, 0.35686183,
        1.        , 1.        , 1.        , 1.        , 1.        ]])
```
**Expected behavior**
It should behave like run with local session.
```
In [1]: import mars.tensor as mt

In [2]: a = mt.random.rand(10, 10)

In [3]: a[:5, :5] = mt.ones((5, 5))

In [4]: a[5:10, 5:10] = mt.ones((5, 5)) * 2

In [5]: a.execute()
Out[5]: 
array([[1.        , 1.        , 1.        , 1.        , 1.        ,
        0.32666468, 0.17133841, 0.78110928, 0.3470744 , 0.75992179],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.80793372, 0.28223076, 0.48451418, 0.20444977, 0.01312504],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.3446098 , 0.15119387, 0.53543721, 0.64302594, 0.24665612],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.36354404, 0.64536075, 0.70544883, 0.35930292, 0.18954646],
       [1.        , 1.        , 1.        , 1.        , 1.        ,
        0.27617278, 0.70748526, 0.5806906 , 0.23309319, 0.2004623 ],
       [0.52383834, 0.33428529, 0.72989358, 0.88056503, 0.9267791 ,
        2.        , 2.        , 2.        , 2.        , 2.        ],
       [0.43696731, 0.04261789, 0.7842462 , 0.79119168, 0.67635289,
        2.        , 2.        , 2.        , 2.        , 2.        ],
       [0.5069902 , 0.02869844, 0.96092322, 0.56741621, 0.00999291,
        2.        , 2.        , 2.        , 2.        , 2.        ],
       [0.47154552, 0.17418114, 0.94618458, 0.64845816, 0.37390886,
        2.        , 2.        , 2.        , 2.        , 2.        ],
       [0.91325706, 0.28710271, 0.46964672, 0.92901795, 0.82593861,
        2.        , 2.        , 2.        , 2.        , 2.        ]])
```




Discussion:

Resolving patch:
diff --git a/mars/scheduler/graph.py b/mars/scheduler/graph.py
--- a/mars/scheduler/graph.py
+++ b/mars/scheduler/graph.py
@@ -159,7 +159,8 @@ def __init__(self, session_id, graph_key, serialized_tensor_graph,
         self._op_key_to_chunk = defaultdict(list)
 
         self._resource_actor = None
-        self._tensor_to_tiled = defaultdict(list)
+        self._tensor_key_opid_to_tiled = defaultdict(list)
+        self._tensor_key_to_opid = dict()
         self._terminal_chunk_op_tensor = defaultdict(set)
         self._terminated_tensors = set()
         self._operand_infos = dict()
@@ -249,7 +250,7 @@ def _detect_cancel(callback=None):
             _detect_cancel(self.stop_graph)
         except ExecutionInterrupted:
             pass
-        except:
+        except:  # noqa: E722
             logger.exception('Failed to start graph execution.')
             self.stop_graph()
             self.state = GraphState.FAILED
@@ -270,6 +271,7 @@ def stop_graph(self):
             self.state = GraphState.CANCELLED
             return
 
+        has_stopping = False
         for chunk in chunk_graph:
             if chunk.op.key not in self._operand_infos:
                 continue
@@ -279,7 +281,10 @@ def stop_graph(self):
                 op_uid = OperandActor.gen_uid(self._session_id, chunk.op.key)
                 scheduler_addr = self.get_scheduler(op_uid)
                 ref = self.ctx.actor_ref(op_uid, address=scheduler_addr)
+                has_stopping = True
                 ref.stop_operand(_tell=True)
+        if not has_stopping:
+            self.state = GraphState.CANCELLED
 
     def reload_chunk_graph(self):
         """
@@ -328,24 +333,25 @@ def prepare_graph(self, compose=True):
 
         key_to_chunk = {c.key: c for c in chunk_graph}
 
-        tensor_to_tiled = self._tensor_to_tiled
+        tensor_key_opid_to_tiled = self._tensor_key_opid_to_tiled
 
         for t in tensor_graph:
-            if t.key not in tensor_to_tiled:
+            self._tensor_key_to_opid[t.key] = t.op.id
+            if (t.key, t.op.id) not in tensor_key_opid_to_tiled:
                 continue
-            t._chunks = [key_to_chunk[k] for k in [tensor_to_tiled[t.key][-1]]]
+            t._chunks = [key_to_chunk[k] for k in [tensor_key_opid_to_tiled[(t.key, t.op.id)][-1]]]
 
         tq = deque()
         for t in tensor_graph:
-            if t.inputs and not all(ti.key in tensor_to_tiled for ti in t.inputs):
+            if t.inputs and not all((ti.key, ti.op.id) in tensor_key_opid_to_tiled for ti in t.inputs):
                 continue
             tq.append(t)
 
         while tq:
             tensor = tq.popleft()
-            if not tensor.is_coarse() or tensor.key in tensor_to_tiled:
+            if not tensor.is_coarse() or (tensor.key, tensor.op.id) in tensor_key_opid_to_tiled:
                 continue
-            inputs = [tensor_to_tiled[it.key][-1] for it in tensor.inputs or ()]
+            inputs = [tensor_key_opid_to_tiled[(it.key, it.op.id)][-1] for it in tensor.inputs or ()]
 
             op = tensor.op.copy()
             _ = op.new_tensors(inputs, [o.shape for o in tensor.op.outputs],  # noqa: F841
@@ -366,7 +372,7 @@ def prepare_graph(self, compose=True):
                         total_tiled.append(td)
 
                 tiled = total_tiled[j]
-                tensor_to_tiled[t.key].append(tiled)
+                tensor_key_opid_to_tiled[(t.key, t.op.id)].append(tiled)
 
                 # add chunks to fine grained graph
                 q = deque([tiled_c.data for tiled_c in tiled.chunks])
@@ -384,16 +390,16 @@ def prepare_graph(self, compose=True):
                             q.append(ic)
                         chunk_graph.add_edge(ic, c)
 
-                for succ in tensor_graph.iter_successors(t):
-                    if any(t.key not in tensor_to_tiled for t in succ.inputs):
+                for succ in tensor_graph.successors(t):
+                    if any((t.key, t.op.id) not in tensor_key_opid_to_tiled for t in succ.inputs):
                         continue
                     tq.append(succ)
 
         # record the chunk nodes in graph
         reserve_chunk = set()
         result_chunk_keys = list()
-        for tk in self._target_tensor_chunk_ops:
-            for n in [c.data for t in tensor_to_tiled[tk] for c in t.chunks]:
+        for tk_topid in tensor_key_opid_to_tiled:
+            for n in [c.data for t in tensor_key_opid_to_tiled[tk_topid] for c in t.chunks]:
                 result_chunk_keys.append(n.key)
                 dq_predecessors = deque([n])
                 while dq_predecessors:
@@ -410,10 +416,10 @@ def prepare_graph(self, compose=True):
         if compose:
             chunk_graph.compose(keys=result_chunk_keys)
 
-        for tk in tensor_to_tiled:
+        for tk, topid in tensor_key_opid_to_tiled:
             if tk not in self._target_tensor_chunk_ops:
                 continue
-            for n in tensor_to_tiled[tk][-1].chunks:
+            for n in tensor_key_opid_to_tiled[(tk, topid)][-1].chunks:
                 self._terminal_chunk_op_tensor[n.op.key].add(tk)
                 self._target_tensor_chunk_ops[tk].add(n.op.key)
 
@@ -738,9 +744,13 @@ def calc_stats(self):
                 transposed[state].append(data_src[op][sid])
         return ops, transposed, finished * 100.0 / total_count
 
+    def _get_tensor_by_key(self, key):
+        tid = self._tensor_key_to_opid[key]
+        return self._tensor_key_opid_to_tiled[(key, tid)][-1]
+
     def free_tensor_data(self, tensor_key):
         from .operand import OperandActor
-        tiled_tensor = self._tensor_to_tiled[tensor_key][-1]
+        tiled_tensor = self._get_tensor_by_key(tensor_key)
         for chunk in tiled_tensor.chunks:
             op_uid = OperandActor.gen_uid(self._session_id, chunk.op.key)
             scheduler_addr = self.get_scheduler(op_uid)
@@ -751,7 +761,7 @@ def build_tensor_merge_graph(self, tensor_key):
         from ..tensor.expressions.merge.concatenate import TensorConcatenate
         from ..tensor.expressions.datasource import TensorFetchChunk
 
-        tiled_tensor = self._tensor_to_tiled[tensor_key][-1]
+        tiled_tensor = self._get_tensor_by_key(tensor_key)
         graph = DAG()
         if len(tiled_tensor.chunks) == 1:
             # only one chunk, just trigger fetch
@@ -777,7 +787,7 @@ def fetch_tensor_result(self, tensor_key):
         from ..worker.transfer import ResultSenderActor
 
         # TODO for test
-        tiled_tensor = self._tensor_to_tiled[tensor_key][-1]
+        tiled_tensor = self._get_tensor_by_key(tensor_key)
         if tensor_key not in self._terminated_tensors:
             return None
 


Test patch:
diff --git a/mars/scheduler/tests/test_graph.py b/mars/scheduler/tests/test_graph.py
--- a/mars/scheduler/tests/test_graph.py
+++ b/mars/scheduler/tests/test_graph.py
@@ -23,18 +23,13 @@
 
 
 class Test(unittest.TestCase):
-
-    def testGraphActor(self):
+    def run_expr_suite(self, expr):
         session_id = str(uuid.uuid4())
         graph_key = str(uuid.uuid4())
 
-        arr = mt.random.randint(10, size=(10, 8), chunk_size=4)
-        arr_add = mt.random.randint(10, size=(10, 8), chunk_size=4)
-        arr2 = arr + arr_add
-
-        graph = arr2.build_graph(compose=False)
+        graph = expr.build_graph(compose=False)
         serialized_graph = serialize_graph(graph)
-        chunked_graph = arr2.build_graph(compose=False, tiled=True)
+        chunked_graph = expr.build_graph(compose=False, tiled=True)
 
         addr = '127.0.0.1:%d' % get_next_port()
         with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool:
@@ -76,16 +71,25 @@ def testGraphActor(self):
             graph_ref.create_operand_actors(_clean_io_meta=False)
             op_infos = graph_ref.get_operand_info()
 
+            orig_metas = dict()
+            for n in fetched_graph:
+                try:
+                    meta = orig_metas[n.op.key]
+                except KeyError:
+                    meta = orig_metas[n.op.key] = dict(
+                        predecessors=set(), successors=set(), input_chunks=set(), chunks=set()
+                    )
+                meta['predecessors'].update([pn.op.key for pn in fetched_graph.iter_predecessors(n)])
+                meta['successors'].update([sn.op.key for sn in fetched_graph.iter_successors(n)])
+                meta['input_chunks'].update([pn.key for pn in fetched_graph.iter_predecessors(n)])
+                meta['chunks'].update([c.key for c in n.op.outputs])
+
             for n in fetched_graph:
                 self.assertEqual(op_infos[n.op.key]['op_name'], type(n.op).__name__)
 
                 io_meta = op_infos[n.op.key]['io_meta']
-                orig_io_meta = dict(
-                    predecessors=list(set(pn.op.key for pn in fetched_graph.iter_predecessors(n))),
-                    successors=list(set(sn.op.key for sn in fetched_graph.iter_successors(n))),
-                    input_chunks=list(set(pn.key for pn in fetched_graph.iter_predecessors(n))),
-                    chunks=list(c.key for c in n.op.outputs),
-                )
+                orig_io_meta = orig_metas[n.op.key]
+
                 self.assertSetEqual(set(io_meta['predecessors']), set(orig_io_meta['predecessors']))
                 self.assertSetEqual(set(io_meta['successors']), set(orig_io_meta['successors']))
                 self.assertSetEqual(set(io_meta['input_chunks']), set(orig_io_meta['input_chunks']))
@@ -93,139 +97,29 @@ def testGraphActor(self):
 
                 self.assertEqual(op_infos[n.op.key]['output_size'], sum(ch.nbytes for ch in n.op.outputs))
 
-    def testGraphWithSplit(self):
-        session_id = str(uuid.uuid4())
-        graph_key = str(uuid.uuid4())
+    def testGraphActor(self):
+        arr = mt.random.randint(10, size=(10, 8), chunk_size=4)
+        arr_add = mt.random.randint(10, size=(10, 8), chunk_size=4)
+        arr2 = arr + arr_add
+        self.run_expr_suite(arr2)
 
+    def testGraphWithSplit(self):
         arr = mt.ones(12, chunk_size=4)
         arr_split = mt.split(arr, 2)
         arr_sum = arr_split[0] + arr_split[1]
+        self.run_expr_suite(arr_sum)
 
-        graph = arr_sum.build_graph(compose=False)
-        serialized_graph = serialize_graph(graph)
-        chunked_graph = arr_sum.build_graph(compose=False, tiled=True)
-
-        addr = '127.0.0.1:%d' % get_next_port()
-        with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool:
-            pool.create_actor(ClusterInfoActor, [pool.cluster_info.address],
-                              uid=ClusterInfoActor.default_name())
-            resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_name())
-            pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_name())
-            pool.create_actor(AssignerActor, uid=AssignerActor.gen_name(session_id))
-            graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph,
-                                          uid=GraphActor.gen_name(session_id, graph_key))
-
-            graph_ref.prepare_graph(compose=False)
-            fetched_graph = graph_ref.get_chunk_graph()
-            self.assertIsNotNone(fetched_graph)
-            self.assertEqual(len(chunked_graph), len(fetched_graph))
-
-            graph_ref.scan_node()
-            op_infos = graph_ref.get_operand_info()
-            for n in fetched_graph:
-                depth = op_infos[n.op.key]['optimize']['depth']
-                self.assertIsNotNone(depth)
-                successor_size = op_infos[n.op.key]['optimize']['successor_size']
-                self.assertIsNotNone(successor_size)
-                descendant_size = op_infos[n.op.key]['optimize']['descendant_size']
-                self.assertIsNotNone(descendant_size)
-
-            resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4)))
-            resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4)))
-
-            graph_ref.place_initial_chunks()
-            op_infos = graph_ref.get_operand_info()
-
-            for n in fetched_graph:
-                if fetched_graph.count_predecessors(n) != 0:
-                    continue
-                target_worker = op_infos[n.op.key]['target_worker']
-                self.assertIsNotNone(target_worker)
-
-            graph_ref.create_operand_actors(_clean_io_meta=False)
-            op_infos = graph_ref.get_operand_info()
-
-            for n in fetched_graph:
-                self.assertEqual(op_infos[n.op.key]['op_name'], type(n.op).__name__)
-
-                io_meta = op_infos[n.op.key]['io_meta']
-                orig_io_meta = dict(
-                    predecessors=list(set(pn.op.key for pn in fetched_graph.iter_predecessors(n))),
-                    successors=list(set(sn.op.key for sn in fetched_graph.iter_successors(n))),
-                    input_chunks=list(set(pn.key for pn in fetched_graph.iter_predecessors(n))),
-                    chunks=list(c.key for c in n.op.outputs),
-                )
-                self.assertSetEqual(set(io_meta['predecessors']), set(orig_io_meta['predecessors']))
-                self.assertSetEqual(set(io_meta['successors']), set(orig_io_meta['successors']))
-                self.assertSetEqual(set(io_meta['input_chunks']), set(orig_io_meta['input_chunks']))
-                self.assertSetEqual(set(io_meta['chunks']), set(orig_io_meta['chunks']))
-
-                self.assertEqual(op_infos[n.op.key]['output_size'], sum(ch.nbytes for ch in n.op.outputs))
-
-    def testSameKey(self, *_):
-        session_id = str(uuid.uuid4())
-        graph_key = str(uuid.uuid4())
-
+    def testSameKey(self):
         arr = mt.ones((5, 5), chunk_size=3)
         arr2 = mt.concatenate((arr, arr))
+        self.run_expr_suite(arr2)
 
-        graph = arr2.build_graph(compose=False)
-        serialized_graph = serialize_graph(graph)
-        chunked_graph = arr2.build_graph(compose=False, tiled=True)
-
-        addr = '127.0.0.1:%d' % get_next_port()
-        with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool:
-            pool.create_actor(ClusterInfoActor, [pool.cluster_info.address],
-                              uid=ClusterInfoActor.default_name())
-            resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_name())
-            pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_name())
-            pool.create_actor(AssignerActor, uid=AssignerActor.gen_name(session_id))
-            graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph,
-                                          uid=GraphActor.gen_name(session_id, graph_key))
-
-            graph_ref.prepare_graph(compose=False)
-            fetched_graph = graph_ref.get_chunk_graph()
-            self.assertIsNotNone(fetched_graph)
-            self.assertEqual(len(chunked_graph), len(fetched_graph))
-
-            graph_ref.scan_node()
-            op_infos = graph_ref.get_operand_info()
-            for n in fetched_graph:
-                depth = op_infos[n.op.key]['optimize']['depth']
-                self.assertIsNotNone(depth)
-                successor_size = op_infos[n.op.key]['optimize']['successor_size']
-                self.assertIsNotNone(successor_size)
-                descendant_size = op_infos[n.op.key]['optimize']['descendant_size']
-                self.assertIsNotNone(descendant_size)
-
-            resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4)))
-            resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4)))
-
-            graph_ref.place_initial_chunks()
-            op_infos = graph_ref.get_operand_info()
+    def testMultipleAdd(self):
+        import numpy as np
+        import operator
+        from mars.compat import reduce
 
-            for n in fetched_graph:
-                if fetched_graph.count_predecessors(n) != 0:
-                    continue
-                target_worker = op_infos[n.op.key]['target_worker']
-                self.assertIsNotNone(target_worker)
-
-            graph_ref.create_operand_actors(_clean_io_meta=False)
-            op_infos = graph_ref.get_operand_info()
-
-            for n in fetched_graph:
-                self.assertEqual(op_infos[n.op.key]['op_name'], type(n.op).__name__)
-
-                io_meta = op_infos[n.op.key]['io_meta']
-                orig_io_meta = dict(
-                    predecessors=list(set(pn.op.key for pn in fetched_graph.iter_predecessors(n))),
-                    successors=list(set(sn.op.key for sn in fetched_graph.iter_successors(n))),
-                    input_chunks=list(set(pn.key for pn in fetched_graph.iter_predecessors(n))),
-                    chunks=list(c.key for c in n.op.outputs),
-                )
-                self.assertSetEqual(set(io_meta['predecessors']), set(orig_io_meta['predecessors']))
-                self.assertSetEqual(set(io_meta['successors']), set(orig_io_meta['successors']))
-                self.assertSetEqual(set(io_meta['input_chunks']), set(orig_io_meta['input_chunks']))
-                self.assertSetEqual(set(io_meta['chunks']), set(orig_io_meta['chunks']))
-
-                self.assertEqual(op_infos[n.op.key]['output_size'], sum(ch.nbytes for ch in n.op.outputs))
+        base_arr = np.random.random((100, 100))
+        a = mt.array(base_arr)
+        sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
+        self.run_expr_suite(sumv)
diff --git a/mars/scheduler/tests/test_main.py b/mars/scheduler/tests/test_main.py
--- a/mars/scheduler/tests/test_main.py
+++ b/mars/scheduler/tests/test_main.py
@@ -12,22 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
-import os
-import sys
-import subprocess
-import uuid
 import json
+import operator
+import os
 import signal
+import subprocess
+import sys
+import time
 import unittest
+import uuid
 
 import numpy as np
 from numpy.testing import assert_array_equal
 import gevent
 
+from mars import tensor as mt
+from mars.compat import reduce
 from mars.serialize.dataserializer import loads
 from mars.config import options
-from mars.tensor.expressions.datasource import ones
 from mars.utils import get_next_port
 from mars.actors.core import new_client
 from mars.scheduler import SessionActor, ResourceActor
@@ -124,7 +126,7 @@ def check_process_statuses(self):
     def wait_for_termination(self, session_ref, graph_key):
         check_time = time.time()
         while True:
-            time.sleep(1)
+            time.sleep(0.1)
             self.check_process_statuses()
             if time.time() - check_time > 60:
                 raise SystemError('Check graph status timeout')
@@ -139,8 +141,8 @@ def testMain(self):
                                                 uid=SessionActor.gen_name(session_id),
                                                 address=scheduler_address,
                                                 session_id=session_id)
-        a = ones((100, 100), chunk_size=30) * 2 * 1 + 1
-        b = ones((100, 100), chunk_size=30) * 2 * 1 + 1
+        a = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
+        b = mt.ones((100, 100), chunk_size=30) * 2 * 1 + 1
         c = (a * b * 2 + 1).sum()
         graph = c.build_graph()
         targets = [c.key]
@@ -163,8 +165,8 @@ def testMain(self):
         state = self.wait_for_termination(session_ref, graph_key)
         self.assertEqual(state, GraphState.FAILED)
 
-        a = ones((100, 50), chunk_size=30) * 2 + 1
-        b = ones((50, 200), chunk_size=30) * 2 + 1
+        a = mt.ones((100, 50), chunk_size=35) * 2 + 1
+        b = mt.ones((50, 200), chunk_size=35) * 2 + 1
         c = a.dot(b)
         graph = c.build_graph()
         targets = [c.key]
@@ -176,3 +178,19 @@ def testMain(self):
         self.assertEqual(state, GraphState.SUCCEEDED)
         result = session_ref.fetch_result(graph_key, c.key)
         assert_array_equal(loads(result), np.ones((100, 200)) * 450)
+
+        base_arr = np.random.random((100, 100))
+        a = mt.array(base_arr)
+        sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
+        graph = sumv.build_graph()
+        targets = [sumv.key]
+        graph_key = uuid.uuid1()
+        session_ref.submit_tensor_graph(json.dumps(graph.to_json()),
+                                        graph_key, target_tensors=targets)
+
+        state = self.wait_for_termination(session_ref, graph_key)
+        self.assertEqual(state, GraphState.SUCCEEDED)
+
+        expected = reduce(operator.add, [base_arr[:10, :10] for _ in range(10)])
+        result = session_ref.fetch_result(graph_key, sumv.key)
+        assert_array_equal(loads(result), expected)