Repository: mars-project/mars

Issue:
[BUG] histogram fails on DataFrames with only one chunk without deterministic shape
**Describe the bug**

Histogram fails on DataFrames with only one chunk without deterministic shape.

Code reproducing this bug:

```python
import mars.dataframe as md, mars.tensor as mt
import numpy as np
rnd = np.random.rand(100, 10)
mdf = md.DataFrame(rnd)
mdf1 = mdf.reset_index().rename(columns={0: 'a'})
ser = mdf1.a.fillna(0)
tp = mt.histogram(ser[np.isfinite(ser)])
tp.execute()
```


Discussion:
Maybe #1869 does not fix this scenario.

Resolving patch:
diff --git a/mars/dataframe/core.py b/mars/dataframe/core.py
--- a/mars/dataframe/core.py
+++ b/mars/dataframe/core.py
@@ -894,6 +894,10 @@ def iteritems(self, batch_size=10000, session=None):
 
     items = iteritems
 
+    def to_dict(self, into=dict, batch_size=10000, session=None):
+        fetch_kwargs = dict(batch_size=batch_size)
+        return self.to_pandas(session=session, fetch_kwargs=fetch_kwargs).to_dict(into=into)
+
 
 class Series(HasShapeTileableEnity, _ToPandasMixin):
     __slots__ = '_cache',
@@ -1026,6 +1030,38 @@ def iteritems(self, batch_size=10000, session=None):
 
     items = iteritems
 
+    def to_dict(self, into=dict, batch_size=10000, session=None):
+        """
+        Convert Series to {label -> value} dict or dict-like object.
+
+        Parameters
+        ----------
+        into : class, default dict
+            The collections.abc.Mapping subclass to use as the return
+            object. Can be the actual class or an empty
+            instance of the mapping type you want.  If you want a
+            collections.defaultdict, you must pass it initialized.
+
+        Returns
+        -------
+        collections.abc.Mapping
+            Key-value representation of Series.
+
+        Examples
+        --------
+        >>> import mars.dataframe as md
+        >>> s = md.Series([1, 2, 3, 4])
+        >>> s.to_dict()
+        {0: 1, 1: 2, 2: 3, 3: 4}
+        >>> from collections import OrderedDict, defaultdict
+        >>> s.to_dict(OrderedDict)
+        OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)])
+        >>> dd = defaultdict(list)
+        >>> s.to_dict(dd)
+        defaultdict(<class 'list'>, {0: 1, 1: 2, 2: 3, 3: 4})
+        """
+        return self._data.to_dict(into=into, batch_size=batch_size, session=session)
+
     def to_frame(self, name=None):
         """
         Convert Series to DataFrame.
diff --git a/mars/tensor/base/flatten.py b/mars/tensor/base/flatten.py
--- a/mars/tensor/base/flatten.py
+++ b/mars/tensor/base/flatten.py
@@ -60,4 +60,4 @@ def flatten(a, order='C'):
     new_shape = calc_shape(a.size, -1)
     tensor_order = get_order(order, a.order)
     op = TensorReshape(new_shape, dtype=a.dtype, create_view=False)
-    return op(a, order=tensor_order)
+    return op(a, order=tensor_order, out_shape=new_shape)
diff --git a/mars/tensor/reshape/reshape.py b/mars/tensor/reshape/reshape.py
--- a/mars/tensor/reshape/reshape.py
+++ b/mars/tensor/reshape/reshape.py
@@ -86,8 +86,8 @@ def on_input_modify(self, new_input):
         op = self.copy().reset_key()
         return op(new_input)
 
-    def __call__(self, a, order):
-        return self.new_tensor([a], self._newshape, order=order)
+    def __call__(self, a, order, out_shape):
+        return self.new_tensor([a], out_shape, order=order)
 
     @staticmethod
     def _gen_reshape_rechunk_nsplits(old_shape, new_shape, nsplits):
@@ -240,7 +240,7 @@ def tile(cls, op):
             # -1 exists in newshape and input tensor has unknown shape
             # recalculate new shape
             shape = tuple(-1 if np.isnan(s) else s for s in tensor.shape)
-            newshape = calc_shape(in_tensor.size, shape)
+            op._newshape = newshape = calc_shape(in_tensor.size, shape)
             tensor._shape = newshape
 
         if op.order == 'F':
@@ -515,9 +515,9 @@ def reshape(a, newshape, order='C'):
         new_shape = [newshape] if isinstance(newshape, int) else list(newshape)
         # if -1 exists in newshape, just treat it as unknown shape
         new_shape = [s if s != -1 else np.nan for s in new_shape]
-        newshape = tuple(new_shape)
+        out_shape = tuple(new_shape)
     else:
-        newshape = calc_shape(a.size, newshape)
+        out_shape = newshape = calc_shape(a.size, newshape)
         if a.size != np.prod(newshape):
             raise ValueError(f'cannot reshape array of size {a.size} into shape {newshape}')
 
@@ -526,12 +526,15 @@ def reshape(a, newshape, order='C'):
     if a.shape == newshape and tensor_order == a.order:
         # does not need to reshape
         return a
-    return _reshape(a, newshape, order=order, tensor_order=tensor_order)
+    return _reshape(a, newshape, order=order,
+                    tensor_order=tensor_order, out_shape=out_shape)
 
 
-def _reshape(a, newshape, order='C', tensor_order=None):
+def _reshape(a, newshape, order='C', tensor_order=None, out_shape=None):
     if tensor_order is None:
         tensor_order = get_order(order, a.order, available_options='CFA')
     op = TensorReshape(newshape, order, dtype=a.dtype,
                        create_view=tensor_order == a.order)
-    return op(a, tensor_order)
+    if out_shape is None:
+        out_shape = newshape
+    return op(a, tensor_order, out_shape)


Test patch:
diff --git a/mars/tensor/reshape/tests/test_reshape_execute.py b/mars/tensor/reshape/tests/test_reshape_execute.py
--- a/mars/tensor/reshape/tests/test_reshape_execute.py
+++ b/mars/tensor/reshape/tests/test_reshape_execute.py
@@ -57,14 +57,16 @@ def testReshapeExecution(self):
         self.assertFalse(res.flags['C_CONTIGUOUS'])
 
         with self.ctx:
-            data = np.random.RandomState(0).rand(3, 4, 5)
-            x = tensor(data, chunk_size=3)
-            x = x[x[:, 0, 0] < 0.7]
-            y = x.reshape(-1, 20)
-            self.assertTrue(np.isnan(y.shape[0]))
-            res = self.executor.execute_tensors([y])[0]
-            expected = data[data[:, 0, 0] < 0.7].reshape(-1, 20)
-            np.testing.assert_array_equal(res, expected)
+            for chunk_size in [None, 3]:
+                rs = np.random.RandomState(0)
+                data = rs.rand(3, 4, 5)
+                x = tensor(data, chunk_size=chunk_size)
+                x = x[x[:, 0, 0] < 0.7]
+                y = x.reshape(-1, 20)
+                self.assertTrue(np.isnan(y.shape[0]))
+                res = self.executor.execute_tensors([y])[0]
+                expected = data[data[:, 0, 0] < 0.7].reshape(-1, 20)
+                np.testing.assert_array_equal(res, expected)
 
     def testShuffleReshapeExecution(self):
         a = ones((31, 27), chunk_size=10)
diff --git a/mars/tests/test_session.py b/mars/tests/test_session.py
--- a/mars/tests/test_session.py
+++ b/mars/tests/test_session.py
@@ -660,6 +660,9 @@ def testIter(self):
 
         self.assertEqual(i, len(raw_data))
 
+        # test to_dict
+        self.assertEqual(s.to_dict(), raw_data.to_dict())
+
     def testNamed(self):
         rs = np.random.RandomState(0)
         raw = rs.rand(10, 10)