Repository: mars-project/mars

Issue:
[BUG] Creating DataFrame from pandas generates many empty chunks
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Creating DataFrame from pandas generates many empty chunks.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and pandas
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [1]: import pandas as pd

In [2]: import faker

In [3]: faker = faker.Faker('zh_CN')

In [4]: all_names = [faker.name() for _ in range(2000)]

In [5]: size = 2000_0000

In [7]: import numpy as np

In [8]: pdf = pd.DataFrame({'name': np.random.choice(all_names, size=size), 'age
   ...: ': np.random.randint(3, 60, size=size)})

In [9]: import mars.dataframe as md

In [10]: df = md.DataFrame(pdf)

In [11]: from mars.core import tile

In [12]: tdf = tile(df)

In [13]: tdf.nsplits
Out[13]: 
((1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  1416868,
  163848),
 (2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))

```



Discussion:

Resolving patch:
diff --git a/mars/dataframe/datasource/dataframe.py b/mars/dataframe/datasource/dataframe.py
--- a/mars/dataframe/datasource/dataframe.py
+++ b/mars/dataframe/datasource/dataframe.py
@@ -14,11 +14,14 @@
 
 import itertools
 
+import pandas as pd
+
 from ... import opcodes as OperandDef
 from ...config import options
 from ...core import OutputType
 from ...serialization.serializables import DataFrameField, SeriesField
 from ...tensor.utils import get_chunk_slices
+from ...utils import estimate_pandas_size
 from ..utils import decide_dataframe_chunk_sizes, parse_index, is_cudf
 from ..operands import DataFrameOperand, DataFrameOperandMixin
 
@@ -61,7 +64,10 @@ def tile(cls, op: "DataFrameDataSource"):
         df = op.outputs[0]
         raw_df = op.data
 
-        memory_usage = raw_df.memory_usage(index=False, deep=True)
+        # estimate column memory usage instead of calling df.memory_usage(deep=True)
+        memory_usage = pd.Series(
+            {c: estimate_pandas_size(s) for c, s in raw_df.iteritems()}
+        )
         chunk_size = df.extra_params.raw_chunk_size or options.chunk_size
         chunk_size = decide_dataframe_chunk_sizes(df.shape, chunk_size, memory_usage)
         chunk_size_idxes = (range(len(size)) for size in chunk_size)
diff --git a/mars/dataframe/utils.py b/mars/dataframe/utils.py
--- a/mars/dataframe/utils.py
+++ b/mars/dataframe/utils.py
@@ -201,10 +201,10 @@ def decide_dataframe_chunk_sizes(shape, chunk_size, memory_usage):
             int(np.power(max_chunk_size / nbytes_occupied, 1 / float(nleft))), 1
         )
 
-        if col_left_size == 0:
+        if col_left_size == 0 and not col_chunk_size:
             col_chunk_size.append(0)
 
-        if row_left_size == 0:
+        if row_left_size == 0 and not row_chunk_size:
             row_chunk_size.append(0)
 
         # check col first


Test patch:
diff --git a/mars/dataframe/datasource/tests/test_datasource.py b/mars/dataframe/datasource/tests/test_datasource.py
--- a/mars/dataframe/datasource/tests/test_datasource.py
+++ b/mars/dataframe/datasource/tests/test_datasource.py
@@ -15,6 +15,7 @@
 import os
 import tempfile
 import shutil
+import string
 from collections import OrderedDict
 
 import numpy as np
@@ -22,6 +23,7 @@
 import pytest
 
 from .... import tensor as mt
+from ....config import option_context
 from ....core import tile
 from ....tests.core import require_ray
 from ....utils import lazy_import
@@ -138,6 +140,20 @@ def test_from_pandas_dataframe():
     pd.testing.assert_frame_equal(df2.chunks[5].op.data, df2.op.data.iloc[4:, 8:])
     assert df2.chunks[3].index_value._index_value._slice == slice(8, 10, 2)
 
+    raw = pd.DataFrame(
+        {
+            "a": [
+                string.printable[i : i + 15]
+                for i in np.random.randint(len(string.printable), size=100)
+            ],
+            "b": np.random.rand(100),
+        }
+    )
+    with option_context({"chunk_store_limit": raw["a"].memory_usage(deep=True) / 10}):
+        df = tile(from_pandas_df(raw))
+        # see GH#2985, empty chunks are wrongly generated
+        assert len([ns for ns in df.nsplits[1] if ns == 0]) == 0
+
 
 def test_from_pandas_series():
     data = pd.Series(np.random.rand(10), name="a")