Repository: mars-project/mars

Issue:
[BUG][TENSOR] mt.split got an error when running in cluster
**Describe the bug**
When submit split tensors to cluster, the ValueError is raised on the scheduler side.

**To Reproduce**
```
In [1]: from mars.deploy.local import new_cluster                                                        

In [2]: cluster = new_cluster(scheduler_n_process=2, worker_n_process=3, web=True)                       

In [3]: import mars.tensor as mt                                                                         

In [4]: t = mt.random.rand(10, 10)

In [5]: mt.split(t, 5).execute()                                                                         
Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 716, in gevent._greenlet.Greenlet.run
  File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 527, in get_executable_operand_dag
    kws=kws)
  File "/Users/hekaisheng/Documents/mars_dev/mars/mars/tensor/expressions/indexing/getitem.py", line 58, in new_chunks
    chunk, indexes = inputs
ValueError: not enough values to unpack (expected 2, got 1)
2018-12-25T02:23:13Z <Greenlet "Greenlet-0" at 0x11a9f47b8: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0x11b154458>> failed with ValueError

```

**Expected behavior**
It should run as with thread-based scheduler on a single machine.
```
In [1]: import mars.tensor as mt                                                                         

In [2]: mt.split(mt.random.rand(10, 10), 5).execute()                                                    
Out[2]: 
[array([[0.97707867, 0.65788903, 0.6474784 , 0.14588068, 0.66482111,
         0.41569224, 0.37568813, 0.54747152, 0.55119096, 0.75261828],
        [0.92090163, 0.33438114, 0.33748729, 0.85393778, 0.85343273,
         0.03576545, 0.25153039, 0.84161614, 0.47162213, 0.44469732]]),
 array([[0.19063479, 0.82669046, 0.80276226, 0.69243507, 0.409638  ,
         0.80037753, 0.06239842, 0.48988705, 0.70018657, 0.08936353],
        [0.45298261, 0.97734578, 0.46953004, 0.6408237 , 0.27882975,
         0.71627508, 0.97158351, 0.24061975, 0.39703134, 0.82116613]]),
 array([[0.08595908, 0.98163836, 0.31229413, 0.9138722 , 0.69957077,
         0.55984328, 0.65914406, 0.0157759 , 0.967717  , 0.63201471],
        [0.89268134, 0.4228981 , 0.49237109, 0.12342419, 0.17950104,
         0.05108572, 0.39138997, 0.75485722, 0.64511224, 0.78004265]]),
 array([[0.45108104, 0.96795335, 0.02066064, 0.14883567, 0.60484485,
         0.27167651, 0.66651133, 0.11039622, 0.32972396, 0.27723386],
        [0.5275256 , 0.16618801, 0.97140139, 0.53816923, 0.23349218,
         0.94030826, 0.38716731, 0.88068514, 0.69808024, 0.22292709]]),
 array([[0.52674368, 0.96576199, 0.212461  , 0.70662301, 0.65729539,
         0.9263901 , 0.48157777, 0.81997881, 0.8481292 , 0.7402317 ],
        [0.84808454, 0.2683585 , 0.21626634, 0.68554189, 0.17512773,
         0.84865071, 0.42345878, 0.7005781 , 0.13699969, 0.8125516 ]])]
```



Discussion:

Resolving patch:
diff --git a/mars/tensor/execution/indexing.py b/mars/tensor/execution/indexing.py
--- a/mars/tensor/execution/indexing.py
+++ b/mars/tensor/execution/indexing.py
@@ -32,7 +32,7 @@ def _index(ctx, chunk):
 def _index_set_value(ctx, chunk):
     indexes = [ctx[index.key] if hasattr(index, 'key') else index
                for index in chunk.op.indexes]
-    input = ctx[chunk.inputs[0].key]
+    input = ctx[chunk.inputs[0].key].copy()
     value = ctx[chunk.op.value.key] if hasattr(chunk.op.value, 'key') else chunk.op.value
     if hasattr(input, 'flags') and not input.flags.writeable:
         input.setflags(write=True)
diff --git a/mars/tensor/expressions/indexing/getitem.py b/mars/tensor/expressions/indexing/getitem.py
--- a/mars/tensor/expressions/indexing/getitem.py
+++ b/mars/tensor/expressions/indexing/getitem.py
@@ -17,6 +17,7 @@
 from numbers import Integral
 import operator
 import itertools
+import contextlib
 
 import numpy as np
 
@@ -34,34 +35,39 @@ class TensorIndex(Index, TensorOperandMixin):
     def __init__(self, dtype=None, sparse=False, **kw):
         super(TensorIndex, self).__init__(_dtype=dtype, _sparse=sparse, **kw)
 
-    @classmethod
-    def _handle_inputs(cls, inputs):
-        tensor, indexes = inputs
-        indexes_inputs = [ind for ind in indexes if isinstance(ind, (BaseWithKey, Entity))]
-        return [tensor] + indexes_inputs
-
-    def _set_inputs(self, inputs):
-        super(TensorIndex, self)._set_inputs(inputs)
-        self._input = self._inputs[0]
-        indexes_iter = iter(self._inputs[1:])
-        new_indexes = [next(indexes_iter) if isinstance(index, (BaseWithKey, Entity)) else index
+    @contextlib.contextmanager
+    def _handle_params(self, inputs, indexes):
+        """
+        Index operator is special, it has additional parameter `indexes` which may also be tensor type,
+        normally, this indexes is provided when called by `tile` or `TensorIndex.__call__`, however, calls
+        in `GraphActor.get_executable_operand_dag` only provide inputs, in such situation, we need get `indexes`
+        from operand itself and replace tensor-liked indexes by new one in `inputs`.
+        """
+        if indexes is not None:
+            indexes_inputs = [ind for ind in indexes if isinstance(ind, (BaseWithKey, Entity))]
+            inputs = inputs + indexes_inputs
+        yield inputs
+
+        if indexes is not None:
+            self._indexes = indexes
+
+        inputs_iter = iter(self._inputs[1:])
+        new_indexes = [next(inputs_iter) if isinstance(index, (BaseWithKey, Entity)) else index
                        for index in self._indexes]
         self._indexes = new_indexes
 
     def new_tensors(self, inputs, shape, **kw):
-        tensor, indexes = inputs
-        self._indexes = indexes
-        inputs = self._handle_inputs(inputs)
-        return super(TensorIndex, self).new_tensors(inputs, shape, **kw)
+        indexes = kw.pop('indexes', None)
+        with self._handle_params(inputs, indexes) as mix_inputs:
+            return super(TensorIndex, self).new_tensors(mix_inputs, shape, **kw)
 
     def new_chunks(self, inputs, shape, **kw):
-        chunk, indexes = inputs
-        self._indexes = indexes
-        inputs = self._handle_inputs(inputs)
-        return super(TensorIndex, self).new_chunks(inputs, shape, **kw)
+        indexes = kw.pop('indexes', None)
+        with self._handle_params(inputs, indexes) as mix_inputs:
+            return super(TensorIndex, self).new_chunks(mix_inputs, shape, **kw)
 
     def __call__(self, a, index, shape):
-        return self.new_tensor([a, index], shape)
+        return self.new_tensor([a], shape, indexes=index)
 
     @classmethod
     def tile(cls, op):
@@ -172,14 +178,14 @@ def tile(cls, op):
 
             chunk_input = in_tensor.cix[tuple(chunk_idx)]
             chunk_op = op.copy().reset_key()
-            chunk = chunk_op.new_chunk([chunk_input, chunk_index], tuple(chunk_shape), index=output_idx)
+            chunk = chunk_op.new_chunk([chunk_input], tuple(chunk_shape), indexes=chunk_index, index=output_idx)
             out_chunks.append(chunk)
 
         nsplits = [tuple(c.shape[i] for c in out_chunks
                          if all(idx == 0 for j, idx in enumerate(c.index) if j != i))
                    for i in range(len(out_chunks[0].shape))]
         new_op = op.copy().reset_key()
-        tensor = new_op.new_tensor([op.input, op.indexes], tensor.shape, chunks=out_chunks, nsplits=nsplits)
+        tensor = new_op.new_tensor([op.input], tensor.shape, indexes=op.indexes, chunks=out_chunks, nsplits=nsplits)
 
         if len(to_concat_axis_index) > 1:
             raise NotImplementedError
@@ -204,11 +210,11 @@ def tile(cls, op):
                     axis=axis, dtype=chunks[0].dtype, sparse=chunks[0].op.sparse)
                 concat_chunk = concat_chunk_op.new_chunk(chunks, tuple(s), index=new_idx)
                 out_chunk_op = TensorIndex(dtype=concat_chunk.dtype, sparse=concat_chunk.op.sparse)
-                out_chunk = out_chunk_op.new_chunk([concat_chunk, indexobj], tuple(s), index=new_idx)
+                out_chunk = out_chunk_op.new_chunk([concat_chunk], tuple(s), indexes=indexobj, index=new_idx)
                 output_chunks.append(out_chunk)
 
             new_op = tensor.op.copy()
-            tensor = new_op.new_tensor([op.input, op.indexes], tuple(output_shape),
+            tensor = new_op.new_tensor([op.input], tuple(output_shape), indexes=op.indexes,
                                        chunks=output_chunks, nsplits=output_nsplits)
 
         return [tensor]
diff --git a/mars/tensor/expressions/indexing/setitem.py b/mars/tensor/expressions/indexing/setitem.py
--- a/mars/tensor/expressions/indexing/setitem.py
+++ b/mars/tensor/expressions/indexing/setitem.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from numbers import Integral
+import contextlib
 
 import numpy as np
 
@@ -30,41 +31,45 @@ class TensorIndexSetValue(IndexSetValue, TensorOperandMixin):
     def __init__(self, dtype=None, sparse=False, **kw):
         super(TensorIndexSetValue, self).__init__(_dtype=dtype, _sparse=sparse, **kw)
 
-    @classmethod
-    def _handle_inputs(cls, inputs):
-        tensor, indexes, value = inputs
-        indexes_inputs = [ind for ind in indexes if isinstance(ind, TENSOR_TYPE + CHUNK_TYPE)]
-        inputs = [tensor] + indexes_inputs
-        if isinstance(value, TENSOR_TYPE + CHUNK_TYPE):
-            inputs += [value]
-        return inputs
-
-    def _set_inputs(self, inputs):
-        super(TensorIndexSetValue, self)._set_inputs(inputs)
-        self._input = self._inputs[0]
-        indexes_iter = iter(self._inputs[1:])
-        new_indexes = [next(indexes_iter) if isinstance(index, (BaseWithKey, Entity)) else index
+    @contextlib.contextmanager
+    def _handle_params(self, inputs, indexes, value):
+        """
+        TensorIndexSetValue operator is like Index operand, it has additional parameter `indexes` and `value`, all of
+        them may be tensor type. As explained in TensorIndex, when indexes and value are not provided, we should get
+        from operand itself and replace tensor-liked objects by iterating over inputs.
+        """
+        if indexes is not None and value is not None:
+            indexes_inputs = [ind for ind in indexes if isinstance(ind, TENSOR_TYPE + CHUNK_TYPE)]
+            inputs += indexes_inputs
+            if isinstance(value, TENSOR_TYPE + CHUNK_TYPE):
+                inputs += [value]
+        yield inputs
+
+        if indexes is not None:
+            self._indexes = indexes
+        if value is not None:
+            self._value = value
+        inputs_iter = iter(self._inputs[1:])
+        new_indexes = [next(inputs_iter) if isinstance(index, (BaseWithKey, Entity)) else index
                        for index in self._indexes]
         self._indexes = new_indexes
-        if isinstance(self._value, Entity):
-            self._value = self._value.data
+        if isinstance(self._value, (BaseWithKey, Entity)):
+            self._value = next(inputs_iter)
 
     def new_tensors(self, inputs, shape, **kw):
-        tensor, indexes, value = inputs
-        self._indexes = indexes
-        self._value = value
-        inputs = self._handle_inputs(inputs)
-        return super(TensorIndexSetValue, self).new_tensors(inputs, shape, **kw)
+        indexes = kw.pop('indexes', None)
+        value = kw.pop('value', None)
+        with self._handle_params(inputs, indexes, value) as mix_inputs:
+            return super(TensorIndexSetValue, self).new_tensors(mix_inputs, shape, **kw)
 
     def new_chunks(self, inputs, shape, **kw):
-        chunk, indexes, value = inputs
-        self._indexes = indexes
-        self._value = value
-        inputs = self._handle_inputs(inputs)
-        return super(TensorIndexSetValue, self).new_chunks(inputs, shape, **kw)
+        indexes = kw.pop('indexes', None)
+        value = kw.pop('value', None)
+        with self._handle_params(inputs, indexes, value) as mix_inputs:
+            return super(TensorIndexSetValue, self).new_chunks(mix_inputs, shape, **kw)
 
     def __call__(self, a, index, value):
-        return self.new_tensor([a, index, value], a.shape)
+        return self.new_tensor([a], a.shape, indexes=index, value=value)
 
     @classmethod
     def tile(cls, op):
@@ -73,7 +78,7 @@ def tile(cls, op):
         is_value_tensor = isinstance(value, TENSOR_TYPE)
 
         index_tensor_op = TensorIndex(dtype=tensor.dtype, sparse=op.sparse)
-        index_tensor = index_tensor_op.new_tensor([op.input, op.indexes], tensor.shape).single_tiles()
+        index_tensor = index_tensor_op.new_tensor([op.input], tensor.shape, indexes=op.indexes).single_tiles()
 
         nsplits = index_tensor.nsplits
         if any(any(np.isnan(ns) for ns in nsplit) for nsplit in nsplits):
@@ -92,12 +97,12 @@ def tile(cls, op):
 
             value_chunk = value.cix[index_chunk.index] if is_value_tensor else value
             chunk_op = op.copy().reset_key()
-            out_chunk = chunk_op.new_chunk([chunk, index_chunk.op.indexes, value_chunk],
-                                           chunk.shape, index=chunk.index)
+            out_chunk = chunk_op.new_chunk([chunk], chunk.shape, indexes=index_chunk.op.indexes,
+                                           value=value_chunk, index=chunk.index)
             out_chunks.append(out_chunk)
 
         new_op = op.copy()
-        return new_op.new_tensors([op.input, op.indexes, op.value], tensor.shape,
+        return new_op.new_tensors([op.input], tensor.shape, indexes=op.indexes, value=op.value,
                                   chunks=out_chunks, nsplits=op.input.nsplits)
 
 


Test patch:
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -164,6 +164,36 @@ def testMultipleOutputTensorExecute(self):
                 np.testing.assert_allclose(U_result, U_expected + 1)
                 np.testing.assert_allclose(s_result, s_expectd + 1)
 
+    def testIndexTensorExecute(self):
+        with new_cluster(scheduler_n_process=2, worker_n_process=2) as cluster:
+            session = cluster.session
+
+            a = mt.random.rand(10, 5)
+            idx = slice(0, 5), slice(0, 5)
+            a[idx] = 2
+            a_splits = mt.split(a, 2)
+            r1, r2 = session.run(a_splits[0], a[idx])
+
+            np.testing.assert_array_equal(r1, r2)
+            np.testing.assert_array_equal(r1, np.ones((5, 5)) * 2)
+
+            with new_session(cluster.endpoint) as session2:
+                a = mt.random.rand(10, 5)
+                idx = slice(0, 5), slice(0, 5)
+
+                a[idx] = mt.ones((5, 5)) * 2
+                r = session2.run(a[idx])
+
+                np.testing.assert_array_equal(r, np.ones((5, 5)) * 2)
+
+    def testExecutableTuple(self):
+        with new_cluster(scheduler_n_process=2, worker_n_process=2, web=True) as cluster:
+            with new_session('http://' + cluster._web_endpoint).as_default() as _:
+                a = mt.ones((20, 10), chunk_size=10)
+                u, s, v = (mt.linalg.svd(a)).execute()
+                np.testing.assert_allclose(u.dot(np.diag(s).dot(v)), np.ones((20, 10)))
+
+
     def testGraphFail(self):
         op = SerializeMustFailOperand(f=3)
         tensor = op.new_tensor(None, (3, 3))
diff --git a/mars/tensor/expressions/tests/test_base.py b/mars/tensor/expressions/tests/test_base.py
--- a/mars/tensor/expressions/tests/test_base.py
+++ b/mars/tensor/expressions/tests/test_base.py
@@ -338,7 +338,7 @@ def testIsIn(self):
         self.assertEqual(len(mask.op.test_elements.chunks), 1)
         self.assertIs(mask.chunks[0].inputs[0], element.chunks[0].data)
 
-        element = 2 * arange(4, chunks=1).reshape(2, 2)
+        element = 2 * arange(4, chunk_size=1).reshape(2, 2)
         test_elements = tensor([1, 2, 4, 8], chunk_size=2)
 
         mask = isin(element, test_elements, invert=True)
diff --git a/mars/web/tests/test_api.py b/mars/web/tests/test_api.py
--- a/mars/web/tests/test_api.py
+++ b/mars/web/tests/test_api.py
@@ -170,11 +170,6 @@ def testApi(self):
             value = sess.run(c, timeout=120)
             assert_array_equal(value, va.dot(vb))
 
-            # test test multiple outputs
-            a = mt.random.rand(10, 10)
-            U, s, V, raw = sess.run(list(mt.linalg.svd(a)) + [a])
-            np.testing.assert_allclose(U.dot(np.diag(s).dot(V)), raw)
-
             # check web UI requests
             res = requests.get(service_ep)
             self.assertEqual(res.status_code, 200)
@@ -185,12 +180,6 @@ def testApi(self):
             res = requests.get(service_ep + '/worker')
             self.assertEqual(res.status_code, 200)
 
-        # test default session run with multiple inputs
-        with new_session(service_ep).as_default() as sess:
-            a = mt.ones((20, 10), chunk_size=10)
-            u, s, v = (mt.linalg.svd(a)).execute()
-            np.testing.assert_allclose(u.dot(np.diag(s).dot(v)), np.ones((20, 10)))
-
 
 class MockResponse:
     def __init__(self, status_code, json_text=None, data=None):