Repository: mars-project/mars

Issue:
[BUG] Error when executing groupby with distributed backend
When executing

```python
df = md.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]), columns=['a', 'b', 'c'])
r = session.run(df.groupby('a'))
```

Error occurred:

```
Attempt 1: Unexpected error AttributeError occurred in executing operand b4e4bc5f7b31094fb234d9ea949251a1 in 0.0.0.0:46150
Traceback (most recent call last):
  File "/Users/wenjun.swj/Code/mars/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/worker/calc.py", line 271, in <lambda>
    .then(lambda context_dict: _start_calc(context_dict)) \
  File "/Users/wenjun.swj/Code/mars/mars/worker/calc.py", line 244, in _start_calc
    return self._calc_results(session_id, graph_key, graph, context_dict, chunk_targets)
  File "/Users/wenjun.swj/Code/mars/mars/worker/calc.py", line 192, in _calc_results
    result_sizes.append(calc_data_size(v))
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 302, in calc_data_size
    return dt.shape[0] * sum(dtype.itemsize for dtype in dt.dtypes)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 302, in <genexpr>
    return dt.shape[0] * sum(dtype.itemsize for dtype in dt.dtypes)
AttributeError: 'str' object has no attribute 'itemsize'
```


Discussion:

Resolving patch:
diff --git a/mars/lib/groupby_wrapper.py b/mars/lib/groupby_wrapper.py
--- a/mars/lib/groupby_wrapper.py
+++ b/mars/lib/groupby_wrapper.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 from collections.abc import Iterable
 
 import cloudpickle
@@ -78,6 +79,10 @@ def __getattr__(self, item):
     def __iter__(self):
         return self.groupby_obj.__iter__()
 
+    def __sizeof__(self):
+        return sys.getsizeof(self.obj) \
+            + sys.getsizeof(getattr(self.groupby_obj.grouper, '_cache', None))
+
     @property
     def empty(self):
         return self.obj.empty
diff --git a/mars/utils.py b/mars/utils.py
--- a/mars/utils.py
+++ b/mars/utils.py
@@ -296,7 +296,7 @@ def calc_data_size(dt):
         return max(sys.getsizeof(dt), dt.nbytes)
     if hasattr(dt, 'shape') and len(dt.shape) == 0:
         return 0
-    if hasattr(dt, 'memory_usage'):
+    if hasattr(dt, 'memory_usage') or hasattr(dt, 'groupby_obj'):
         return sys.getsizeof(dt)
     if hasattr(dt, 'dtypes') and hasattr(dt, 'shape'):
         return dt.shape[0] * sum(dtype.itemsize for dtype in dt.dtypes)
diff --git a/mars/worker/execution.py b/mars/worker/execution.py
--- a/mars/worker/execution.py
+++ b/mars/worker/execution.py
@@ -384,7 +384,10 @@ def estimate_graph_finish_time(self, session_id, graph_key, calc_fetch=True, bas
             for c in graph:
                 if not isinstance(c.op, Fetch):
                     break
-                data_size = calc_data_size(c)
+                try:
+                    data_size = calc_data_size(c)
+                except (AttributeError, TypeError, ValueError):
+                    data_size = 0
                 input_size += data_size
                 data_locations = self.storage_client.get_data_locations(session_id, [c.key])[0]
                 if (0, DataStorageDevice.VINEYARD) in data_locations or \


Test patch:
diff --git a/mars/lib/tests/test_lib.py b/mars/lib/tests/test_lib.py
--- a/mars/lib/tests/test_lib.py
+++ b/mars/lib/tests/test_lib.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import unittest
 
 import pandas as pd
@@ -19,6 +20,7 @@
 
 from mars.lib.groupby_wrapper import GroupByWrapper, wrapped_groupby
 from mars.tests.core import assert_groupby_equal
+from mars.utils import calc_data_size
 
 
 class Test(unittest.TestCase):
@@ -35,6 +37,8 @@ def testGroupByWrapper(self):
         assert_groupby_equal(grouped, df.groupby(level=0))
         self.assertEqual(grouped.shape, (8, 4))
         self.assertTrue(grouped.is_frame)
+        self.assertGreater(sys.getsizeof(grouped), sys.getsizeof(grouped.groupby_obj))
+        self.assertGreater(calc_data_size(grouped), sys.getsizeof(grouped.groupby_obj))
 
         grouped = GroupByWrapper.from_tuple(wrapped_groupby(df, level=0).C.to_tuple())
         assert_groupby_equal(grouped, df.groupby(level=0).C)