Repository: mars-project/mars

Issue:
[BUG] mt.where() Found that there are still problems with some data
```python
self.l2r_arr
Out[16]: 
array([[1.8e+02, 1.8e+02, 4.0e+02, 4.0e+02, 1.6e+02, 1.0e-02],
       [1.8e+02, 1.8e+02, 4.0e+02, 4.0e+02, 1.6e+02, 1.0e-02],
       [1.8e+02, 1.8e+02, 4.0e+02, 4.0e+02, 1.6e+02, 1.0e-02],
       ...,
       [2.6e+02, 1.2e+02, 3.8e+02, 4.0e+02, 1.6e+02, 1.0e-02],
       [2.6e+02, 1.2e+02, 3.8e+02, 4.0e+02, 1.6e+02, 1.0e-02],
       [2.6e+02, 1.2e+02, 3.8e+02, 4.0e+02, 1.6e+02, 1.0e-02]])


mt.where(self.l2r_arr <= 0, 0.01, self.l2r_arr)
```

```
Traceback (most recent call last):
  File "/usr/local/python3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-17-59f7f69c3674>", line 1, in <module>
    mt.where(self.l2r_arr <= 0, 0.01, self.l2r_arr)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/base/where.py", line 182, in where
    return op(condition, x, y, shape=shape)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/utils.py", line 460, in _inner
    return func(*args, **kwargs)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/base/where.py", line 63, in __call__
    return self.new_tensor([condition, x, y], shape)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/operands.py", line 34, in new_tensor
    return self.new_tensors(inputs, shape=shape, dtype=dtype, order=order, **kw)[0]
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/operands.py", line 29, in new_tensors
    output_limit=output_limit, kws=kws, dtype=dtype, order=order, **kw)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/operands.py", line 455, in new_tileables
    ExecutableTuple(tileables).execute(fetch=False)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/core.py", line 770, in execute
    return super().execute(session=session, **kw)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/core.py", line 383, in execute
    return run()
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/core.py", line 378, in run
    session.run(self, **kw)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/session.py", line 506, in run
    result = self._sess.run(*tileables, **kw)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/session.py", line 111, in run
    res = self._executor.execute_tileables(tileables, **kw)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/utils.py", line 460, in _inner
    return func(*args, **kwargs)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/executor.py", line 866, in execute_tileables
    tileables, tileable_graph=tileable_graph)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/utils.py", line 460, in _inner
    return func(*args, **kwargs)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 348, in build
    tileables, tileable_graph=tileable_graph)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/utils.py", line 460, in _inner
    return func(*args, **kwargs)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 262, in build
    self._on_tile_failure(tileable_data.op, exc_info)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 301, in inner
    raise exc_info[1].with_traceback(exc_info[2]) from None
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 242, in build
    tiled = self._tile(tileable_data, tileable_graph)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 337, in _tile
    return super()._tile(tileable_data, tileable_graph)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 201, in _tile
    tds[0]._inplace_tile()
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/core.py", line 169, in _inplace_tile
    return handler.inplace_tile(self)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 136, in inplace_tile
    dispatched = self.dispatch(to_tile.op)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/utils.py", line 460, in _inner
    return func(*args, **kwargs)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tiles.py", line 119, in dispatch
    tiled = op_cls.tile(op)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/base/where.py", line 68, in tile
    inputs = unify_chunks(*[(input, list(range(input.ndim))[::-1]) for input in op.inputs])
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/utils.py", line 495, in unify_chunks
    return unify_nsplits(*tensor_axes)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/utils.py", line 483, in unify_nsplits
    res.append(t.rechunk(new_chunk)._inplace_tile())
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/rechunk/rechunk.py", line 104, in rechunk
    chunk_size = get_nsplits(tensor, chunk_size, tensor.dtype.itemsize)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/rechunk/core.py", line 38, in get_nsplits
    return decide_chunk_sizes(tileable.shape, chunk_size, itemsize)
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/utils.py", line 560, in decide_chunk_sizes
    return normalize_chunk_sizes(shape, tuple(chunk_size[j] for j in range(len(shape))))
  File "/usr/local/python3/lib/python3.6/site-packages/pymars-0.7.0a7-py3.6-linux-x86_64.egg/mars/tensor/utils.py", line 66, in normalize_chunk_sizes
    raise ValueError('chunks shape should be of the same length, '
ValueError: chunks shape should be of the same length, got shape: 7, chunks: (6,)
```


Discussion:
Please provide minimal code reproducing this error stack.
> Please provide minimal code reproducing this error stack.

I probably know the problem. I found that it was the problem. I first deleted a column and then inserted a column. I found that it was not inserted, but the shape changed.

``` Python
>>> import mars.tensor as mt
>>> a = mt.array([[1, 1], [2, 2], [3, 3]])
>>> last_point = a[:,0]
>>> b = mt.delete(a, 0, axis=1)
>>> b
Tensor <op=TensorDelete, shape=(3, 1), key=d03e14c3791cbe24e0421270ec46501c>
>>> b.execute()
array([[1],
       [2],
       [3]])
>>> b = mt.insert(b, a.shape[1] - 1, values=last_point, axis=1)
>>> b
Tensor <op=TensorInsert, shape=(3, 2), key=8355d492230319f87ee50622b12807b4>
>>> b.execute()
array([[1],
       [2],
       [3]])
>>> b.shape()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: 'tuple' object is not callable
>>> b.execute()
array([[1],
       [2],
       [3]])
>>> b.execute().shape
(3, 2)
>>> last_point
Tensor <op=TensorIndex, shape=(3,), key=21be4b37db8a06fd8e62257c49abbf3b>
>>> last_point.execute()
array([1, 2, 3])
```



Thank you for your feedback, will fix it ASAP, cc @hekaisheng .

Resolving patch:
diff --git a/mars/tensor/base/insert.py b/mars/tensor/base/insert.py
--- a/mars/tensor/base/insert.py
+++ b/mars/tensor/base/insert.py
@@ -18,7 +18,7 @@
 
 from ... import opcodes as OperandDef
 from ...core import Base, Entity
-from ...serialize import Int32Field, Int64Field, AnyField, KeyField
+from ...serialize import Int32Field, TupleField, AnyField, KeyField
 from ...tiles import TilesError
 from ...utils import check_chunks_unknown_shape
 from ..datasource import tensor as astensor
@@ -35,12 +35,12 @@ class TensorInsert(TensorHasInput, TensorOperandMixin):
     _input = KeyField('input')
 
     # for chunk
-    _offset_on_axis = Int64Field('offset_on_axis')
+    _range_on_axis = TupleField('range_on_axis')
 
     def __init__(self, index_obj=None, values=None, axis=None,
-                 offset_on_axis=None, dtype=None, **kw):
+                 range_on_axis=None, dtype=None, **kw):
         super().__init__(_index_obj=index_obj, _values=values,
-                         _axis=axis, _offset_on_axis=offset_on_axis,
+                         _axis=axis, _range_on_axis=range_on_axis,
                          _dtype=dtype, **kw)
 
     @property
@@ -56,8 +56,8 @@ def axis(self):
         return self._axis
 
     @property
-    def offset_on_axis(self):
-        return self._offset_on_axis
+    def range_on_axis(self):
+        return self._range_on_axis
 
     def _set_inputs(self, inputs):
         super()._set_inputs(inputs)
@@ -92,12 +92,14 @@ def tile(cls, op: 'TensorInsert'):
         if isinstance(index_obj, int):
             splits = inp.nsplits[axis]
             cum_splits = np.cumsum([0] + list(splits))
+            # add 1 for last split
+            cum_splits[-1] = cum_splits[-1] + 1
             in_idx = cum_splits.searchsorted(index_obj, side='right') - 1
             out_chunks = []
             for chunk in inp.chunks:
                 if chunk.index[axis] == in_idx:
                     chunk_op = op.copy().reset_key()
-                    op._index_obj = index_obj - cum_splits[in_idx]
+                    chunk_op._index_obj = index_obj - cum_splits[in_idx]
                     inputs = filter_inputs([chunk, values])
                     shape = tuple(s + calc_object_length(index_obj) if i == axis else s
                                   for i, s in enumerate(chunk.shape))
@@ -114,12 +116,20 @@ def tile(cls, op: 'TensorInsert'):
             for chunk in inp.chunks:
                 chunk_op = op.copy().reset_key()
                 chunk_op._index_obj = index_obj.chunks[0]
-                chunk_op._values = values.chunks[0]
-                chunk_op._offset_on_axis = offset
+                if isinstance(values, (Base, Entity)):
+                    chunk_values = values.chunks[0]
+                else:
+                    chunk_values = values
+                chunk_op._values = chunk_values
+                if chunk.index[axis] + 1 == len(inp.nsplits[axis]):
+                    # the last chunk on axis
+                    chunk_op._range_on_axis = (offset, offset + chunk.shape[axis] + 1)
+                else:
+                    chunk_op._range_on_axis = (offset, offset + chunk.shape[axis])
                 shape = tuple(np.nan if j == axis else s
                               for j, s in enumerate(chunk.shape))
                 inputs = filter_inputs([chunk, index_obj.chunks[0],
-                                        values.chunks[0]])
+                                        chunk_values])
                 out_chunks.append(chunk_op.new_chunk(inputs, shape=shape,
                                                      index=chunk.index))
                 offset += chunk.shape[axis]
@@ -131,6 +141,8 @@ def tile(cls, op: 'TensorInsert'):
                                   index_obj.step or 1)
             splits = inp.nsplits[axis]
             cum_splits = np.cumsum([0] + list(splits))
+            # add 1 for last split
+            cum_splits[-1] = cum_splits[-1] + 1
             chunk_idx_params = [[[], []] for _ in splits]
             for i, int_idx in enumerate(index_obj):
                 in_idx = cum_splits.searchsorted(int_idx, side='right') - 1
@@ -152,7 +164,10 @@ def tile(cls, op: 'TensorInsert'):
                                                              index=chunk.index))
                     elif isinstance(values, (Base, Entity)):
                         chunk_op._values = values.chunks[0]
-                        chunk_op._offset_on_axis = offset
+                        if chunk.index[axis] + 1 == len(inp.nsplits[axis]):
+                            chunk_op._range_on_axis = (offset, offset + chunk.shape[axis] + 1)
+                        else:
+                            chunk_op._range_on_axis = (offset, offset + chunk.shape[axis])
                         out_chunks.append(chunk_op.new_chunk([chunk, values.chunks[0]],
                                                              shape=shape,
                                                              index=chunk.index))
@@ -189,7 +204,7 @@ def execute(cls, ctx, op: 'TensorInsert'):
         inp = ctx[op.input.key]
         index_obj = ctx[op.index_obj.key] if hasattr(op.index_obj, 'key') else op.index_obj
         values = ctx[op.values.key] if hasattr(op.values, 'key') else op.values
-        if op.offset_on_axis is None:
+        if op.range_on_axis is None:
             ctx[op.outputs[0].key] = np.insert(inp, index_obj, values, axis=op.axis)
         else:
             if isinstance(index_obj, slice):
@@ -199,15 +214,18 @@ def execute(cls, ctx, op: 'TensorInsert'):
             else:
                 index_obj = np.array(index_obj)
             values = np.asarray(values)
+
             part_index = [i for i, idx in enumerate(index_obj) if (
-                    (idx >= op.offset_on_axis) and idx < (op.offset_on_axis + inp.shape[op.axis or 0]))]
-            if len(index_obj) == len(values) and (values[0].ndim > 0 or inp.ndim == 1):
+                    (idx >= op.range_on_axis[0]) and idx < op.range_on_axis[1])]
+            if (values.ndim > 0) and \
+                    len(index_obj) == len(values) and \
+                    (values[0].ndim > 0 or inp.ndim == 1):
                 ctx[op.outputs[0].key] = np.insert(
-                    inp, index_obj[part_index] - op.offset_on_axis,
+                    inp, index_obj[part_index] - op.range_on_axis[0],
                     values[part_index], axis=op.axis)
             else:
                 ctx[op.outputs[0].key] = np.insert(
-                    inp, index_obj[part_index] - op.offset_on_axis,
+                    inp, index_obj[part_index] - op.range_on_axis[0],
                     values, axis=op.axis)
 
     def __call__(self, arr, obj, values, shape):


Test patch:
diff --git a/mars/tensor/base/tests/test_base_execute.py b/mars/tensor/base/tests/test_base_execute.py
--- a/mars/tensor/base/tests/test_base_execute.py
+++ b/mars/tensor/base/tests/test_base_execute.py
@@ -1754,6 +1754,18 @@ def testInsertExecution(self):
         np.testing.assert_array_equal(
             np.insert(raw, slice(0, 10), np.arange(10), axis=0), result)
 
+        r12 = mt.insert(a, 10, 5, axis=1)
+        result = self.executor.execute_tensor(r12, concat=True)[0]
+        np.testing.assert_array_equal(np.insert(raw, 10, 5, axis=1), result)
+
+        r13 = mt.insert(a, [2, 10], 5, axis=1)
+        result = self.executor.execute_tensor(r13, concat=True)[0]
+        np.testing.assert_array_equal(np.insert(raw, [2, 10], 5, axis=1), result)
+
+        r14 = mt.insert(a, mt.tensor([2, 20]), 5, axis=0)
+        result = self.executor.execute_tensor(r14, concat=True)[0]
+        np.testing.assert_array_equal(np.insert(raw, [2, 20], 5, axis=0), result)
+
     def testDeleteExecution(self):
         raw = np.random.randint(0, 100, size=(20, 10))
         a = tensor(raw, chunk_size=6)
@@ -1790,3 +1802,7 @@ def testDeleteExecution(self):
         result = self.executor.execute_tensor(r8, concat=True)[0]
         np.testing.assert_array_equal(
             np.delete(raw, [10, 20, 6, 80]), result)
+
+        r9 = mt.delete(a, 9, axis=1)
+        result = self.executor.execute_tensor(r9, concat=True)[0]
+        np.testing.assert_array_equal(np.delete(raw, 9, axis=1), result)