Repository: modin-project/modin

Issue:
DataFrame.groupby on category broken
### System information
- Linux ip-172-16-74-185 4.14.171-105.231.amzn1.x86_64 #1 SMP Thu Feb 27 23:49:15 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
- **Modin version** (`modin.__version__`): 0.7.3
- **Python version**: 3.6.6 |Anaconda, Inc.| (default, Oct  9 2018, 12:34:16)
- **Code we can use to reproduce**:

# test dataframe

data1 = ['Marc','Lori','Sam', 'Alexandra',
         None, np.NaN, 
         'Marc','Lori','Sam', 'Alexandra', 
         'Marc','Lori','Sam', 'Alexandra']
data2 = [1, 1, 1, 1,
        0,0,
        1, 1, 1, 1,
        1, 1, 1, 1]
data3 = [1.0, 1.0, 1.0, 1.0,
        None, np.NaN,
        1.0, 1.0, 1.0, 1.0,
        1.0, 1.0, 1.0, 1.0]
data4 = ['Same','Same','Same', 'Same',
         None, np.NaN, 
         'Same','Same','Same', 'Same', 
         'Same','Same','Same', 'Same']
data5 = ['Diff1','Diff2','Diff3', 'Diff4',
         None, np.NaN, 
         'Diff5','Diff6','Diff7', 'Diff8',
         'Diff1','Diff2','Diff3', 'Diff4',]

df = pandas_impl_module.DataFrame(data={
    'col1': data1,
    'col2': data2,
    'col3': data3,
    'col4': data4,
    'col5': data5
})
#print("col2 input data type = {}".format(type(data2[0])))
df.info()

df.astype({'col1': 'category'}, copy=False)
df.info()

grpby = df.groupby(by=['col1'], as_index=True, sort=False, observed=False)

grpby.groups

grpby.indices

### Describe the problem
when working with category types in the dataframe, this appears to throw something off in groupby
groupby a category column, then try
accessing the groups or indices attribute of the groupby
exception raised
converting these back to strings and re-doing the groupby appears to work around this problem

### Source code / logs
---------------------------------------------------------------------------
>>> grpby.groups

AttributeError                            Traceback (most recent call last)
<ipython-input-54-fc316fedb9d8> in <module>
----> 1 grpby.groups
~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/modin/pandas/groupby.py in __getattr__(self, key)
     99             if key in self._columns:
    100                 return self._default_to_pandas(lambda df: df.__getitem__(key))
--> 101             raise e
    102 
    103     _index_grouped_cache = None

~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/modin/pandas/groupby.py in __getattr__(self, key)
     95         """
     96         try:
---> 97             return object.__getattribute__(self, key)
     98         except AttributeError as e:
     99             if key in self._columns:

~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/modin/pandas/groupby.py in groups(self)
    202     @property
    203     def groups(self):
--> 204         return self._index_grouped
    205 
    206     def min(self, **kwargs):

~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/modin/pandas/groupby.py in __getattr__(self, key)
     99             if key in self._columns:
    100                 return self._default_to_pandas(lambda df: df.__getitem__(key))
--> 101             raise e
    102 
    103     _index_grouped_cache = None

~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/modin/pandas/groupby.py in __getattr__(self, key)
     95         """
     96         try:
---> 97             return object.__getattribute__(self, key)
     98         except AttributeError as e:
     99             if key in self._columns:

~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/modin/pandas/groupby.py in _index_grouped(self)
    130                     by = self._by
    131                 if self._axis == 0:
--> 132                     self._index_grouped_cache = self._index.groupby(by)
    133                 else:
    134                     self._index_grouped_cache = self._columns.groupby(by)

~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/pandas/core/indexes/base.py in groupby(self, values)
   4533             values = values.values
   4534         values = ensure_categorical(values)
-> 4535         result = values._reverse_indexer()
   4536 
   4537         # map to the label

~/anaconda3/envs/run-nsf/lib/python3.6/site-packages/pandas/core/generic.py in __getattr__(self, name)
   5272             if self._info_axis._can_hold_identifiers_and_holds_name(name):
   5273                 return self[name]
-> 5274             return object.__getattribute__(self, name)
   5275 
   5276     def __setattr__(self, name: str, value) -> None:

AttributeError: 'Series' object has no attribute '_reverse_indexer'


Discussion:
There are some other fixes to categorical dtypes in progress in this PR https://github.com/modin-project/modin/pull/1382 but they are not yet finished because some tests still fail.
I reproduced the problem on PR #1382 branch, so changes in it don't resolve this bug.
This is a simple standalone reproducer
```python
import modin.pandas as pd

df = pd.DataFrame({'col1': ['Marc', 'Lori'], 'col2': [1, 1]})
df.astype({'col1': 'category'}, copy=False)
grpby = df.groupby(by=['col1'], as_index=True, sort=False, observed=False)

print(grpby.groups)
print(grpby.indices)
```
This patch adds a test for this bug
```patch
diff --git a/modin/pandas/test/test_groupby.py b/modin/pandas/test/test_groupby.py
index dd3ea96..a690d75 100644
--- a/modin/pandas/test/test_groupby.py
+++ b/modin/pandas/test/test_groupby.py
@@ -126,10 +126,11 @@ def test_mixed_dtypes_groupby(as_index):


 @pytest.mark.parametrize(
-    "by", [[1, 2, 1, 2], lambda x: x % 3, "col1", ["col1", "col2"]]
+    "by", [[1, 2, 1, 2], lambda x: x % 3, "col1", ["col1"], ["col1", "col2"]]
 )
 @pytest.mark.parametrize("as_index", [True, False])
-def test_simple_row_groupby(by, as_index):
+@pytest.mark.parametrize("col1_categories", [True, False])
+def test_simple_row_groupby(by, as_index, col1_categories):
     pandas_df = pandas.DataFrame(
         {
             "col1": [0, 1, 2, 3],
@@ -140,7 +141,14 @@ def test_simple_row_groupby(by, as_index):
         }
     )

+    if col1_categories:
+        pandas_df.astype({"col1": "category"}, copy=False)
+
     ray_df = from_pandas(pandas_df)
+
+    if col1_categories:
+        ray_df.astype({"col1": "category"}, copy=False)
+
     n = 1
     ray_groupby = ray_df.groupby(by=by, as_index=as_index)
     pandas_groupby = pandas_df.groupby(by=by, as_index=as_index)
```
I had to add a separate conversion for `ray_df` DataFrame returned from `from_pandas(pandas_df)` didn't have categorical dtypes. Not sure if it is a bug or should work like this.

as part of this issue #1460 #1461 #1462 were opened, after fixing those issues investigating about broken groupby on categories will continue
**Current state:**
It seems that bug with groupby on categories appears only when we're grouping by multiple columns, and only in the `groupby_reduce` operations. For example there is the output of modin and pandas `any` operations when grouping on multiple categories columns:
```
Modin:
            col2   col3   col4
col1 col5
0    -7    False  False  False
     -6    False  False  False
     -5    False  False  False
     -4     True  False   True
1    -7    False  False  False
     -6    False  False  False
     -5     True  False   True
     -4    False  False  False
2    -7    False  False  False
     -6    False   True   True
     -5    False  False  False
     -4    False  False  False
3    -7     True   True   True
     -6    False  False  False
     -5    False  False  False
     -4    False  False  False


Pandas:
            col2   col3  col4
col1 col5
0    -7      NaN    NaN   NaN
     -6      NaN    NaN   NaN
     -5      NaN    NaN   NaN
     -4     True  False  True
1    -7      NaN    NaN   NaN
     -6      NaN    NaN   NaN
     -5     True  False  True
     -4      NaN    NaN   NaN
2    -7      NaN    NaN   NaN
     -6    False   True  True
     -5      NaN    NaN   NaN
     -4      NaN    NaN   NaN
3    -7     True   True  True
     -6      NaN    NaN   NaN
     -5      NaN    NaN   NaN
     -4      NaN    NaN   NaN
```
That behaviour could be control with `observe` parameter of groupby.

Investigating...

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -2029,6 +2029,11 @@ def _callable_func(self, func, axis, *args, **kwargs):
     )
 
     def groupby_agg(self, by, axis, agg_func, groupby_args, agg_args, drop=False):
+        # since we're going to modify `groupby_args` dict in a `groupby_agg_builder`,
+        # we want to copy it to not propagate these changes into source dict, in case
+        # of unsuccessful end of function
+        groupby_args = groupby_args.copy()
+
         as_index = groupby_args.get("as_index", True)
 
         def groupby_agg_builder(df):
diff --git a/modin/data_management/functions/groupby_function.py b/modin/data_management/functions/groupby_function.py
--- a/modin/data_management/functions/groupby_function.py
+++ b/modin/data_management/functions/groupby_function.py
@@ -40,7 +40,13 @@ def caller(
                 )
             else:
                 qc = query_compiler
+            # since we're going to modify `groupby_args` dict in a `compute_map`,
+            # we want to copy it to not propagate these changes into source dict, in case
+            # of unsuccessful end of function
+            groupby_args = groupby_args.copy()
+
             as_index = groupby_args.get("as_index", True)
+            observed = groupby_args.get("observed", False)
 
             def _map(df, other):
                 def compute_map(df, other):
@@ -48,6 +54,7 @@ def compute_map(df, other):
                     # It is used to make sure that between phases we are constructing the
                     # right index and placing columns in the correct order.
                     groupby_args["as_index"] = True
+                    groupby_args["observed"] = True
                     other = other.squeeze(axis=axis ^ 1)
                     if isinstance(other, pandas.DataFrame):
                         df = pandas.concat(
@@ -57,6 +64,20 @@ def compute_map(df, other):
                     result = map_func(
                         df.groupby(by=other, axis=axis, **groupby_args), **map_args
                     )
+                    # if `other` has category dtype, then pandas will drop that
+                    # column after groupby, inserting it back to correctly process
+                    # reduce phase
+                    if (
+                        drop
+                        and not as_index
+                        and isinstance(other, pandas.Series)
+                        and isinstance(other.dtype, pandas.CategoricalDtype)
+                        and result.index.name is not None
+                        and result.index.name not in result.columns
+                    ):
+                        result.insert(
+                            loc=0, column=result.index.name, value=result.index
+                        )
                     # The _modin_groupby_ prefix indicates that this is the first partition,
                     # and since we may need to insert the grouping data in the reduce phase
                     if (
@@ -82,6 +103,7 @@ def compute_reduce(df):
                     df = df.reset_index(drop=False)
                     # See note above about setting `as_index`
                     groupby_args["as_index"] = as_index
+                    groupby_args["observed"] = observed
                     if other_len > 1:
                         by_part = list(df.columns[0:other_len])
                     else:
@@ -98,7 +120,7 @@ def compute_reduce(df):
                     if isinstance(by_part, str) and by_part in result.columns:
                         if "_modin_groupby_" in by_part and drop:
                             col_name = by_part[len("_modin_groupby_") :]
-                            new_result = result.drop(columns=col_name)
+                            new_result = result.drop(columns=col_name, errors="ignore")
                             new_result.columns = [
                                 col_name if "_modin_groupby_" in c else c
                                 for c in new_result.columns
diff --git a/modin/pandas/groupby.py b/modin/pandas/groupby.py
--- a/modin/pandas/groupby.py
+++ b/modin/pandas/groupby.py
@@ -128,7 +128,7 @@ def _index_grouped(self):
                 }
             else:
                 if isinstance(self._by, type(self._query_compiler)):
-                    by = self._by.to_pandas().squeeze()
+                    by = self._by.to_pandas().squeeze().values
                 else:
                     by = self._by
                 if self._axis == 0:
@@ -433,7 +433,7 @@ def size(self):
             series_result = Series(query_compiler=result._query_compiler)
             # Pandas does not name size() output
             series_result.name = None
-            return series_result
+            return series_result.fillna(0)
         else:
             return DataFrameGroupBy(
                 self._df.T,
@@ -529,12 +529,16 @@ def fillna(self, **kwargs):
         return result
 
     def count(self, **kwargs):
-        return self._wrap_aggregation(
+        result = self._wrap_aggregation(
             type(self._query_compiler).groupby_count,
             lambda df, **kwargs: df.count(**kwargs),
             numeric_only=False,
             **kwargs,
         )
+        # pandas do it in case of Series
+        if isinstance(result, Series):
+            result = result.fillna(0)
+        return result
 
     def pipe(self, func, *args, **kwargs):
         return com.pipe(self, func, *args, **kwargs)


Test patch:
diff --git a/modin/pandas/test/test_groupby.py b/modin/pandas/test/test_groupby.py
--- a/modin/pandas/test/test_groupby.py
+++ b/modin/pandas/test/test_groupby.py
@@ -21,18 +21,28 @@
     check_df_columns_have_nans,
     create_test_dfs,
     eval_general,
+    df_categories_equals,
 )
 
 pd.DEFAULT_NPARTITIONS = 4
 
 
 def modin_df_almost_equals_pandas(modin_df, pandas_df):
-    difference = to_pandas(modin_df) - pandas_df
+    df_categories_equals(modin_df._to_pandas(), pandas_df)
+
+    modin_df = to_pandas(modin_df)
+
+    if hasattr(modin_df, "select_dtypes"):
+        modin_df = modin_df.select_dtypes(exclude=["category"])
+    if hasattr(pandas_df, "select_dtypes"):
+        pandas_df = pandas_df.select_dtypes(exclude=["category"])
+
+    difference = modin_df - pandas_df
     diff_max = difference.max()
     if isinstance(diff_max, pandas.Series):
         diff_max = diff_max.max()
     assert (
-        to_pandas(modin_df).equals(pandas_df)
+        modin_df.equals(pandas_df)
         or diff_max < 0.0001
         or (all(modin_df.isna().all()) and all(pandas_df.isna().all()))
     )
@@ -234,7 +244,8 @@ def test_mixed_dtypes_groupby(as_index):
     ],
 )
 @pytest.mark.parametrize("as_index", [True, False])
-def test_simple_row_groupby(by, as_index):
+@pytest.mark.parametrize("col1_category", [True, False])
+def test_simple_row_groupby(by, as_index, col1_category):
     pandas_df = pandas.DataFrame(
         {
             "col1": [0, 1, 2, 3],
@@ -245,6 +256,9 @@ def test_simple_row_groupby(by, as_index):
         }
     )
 
+    if col1_category:
+        pandas_df = pandas_df.astype({"col1": "category"})
+
     modin_df = from_pandas(pandas_df)
     n = 1
     modin_groupby = modin_df.groupby(by=by, as_index=as_index)
@@ -267,10 +281,10 @@ def test_simple_row_groupby(by, as_index):
     eval_ndim(modin_groupby, pandas_groupby)
     if not check_df_columns_have_nans(modin_df, by):
         # cum* functions produce undefined results for columns with NaNs so we run them only when "by" columns contain no NaNs
-        eval_cumsum(modin_groupby, pandas_groupby)
-        eval_cummax(modin_groupby, pandas_groupby)
-        eval_cummin(modin_groupby, pandas_groupby)
-        eval_cumprod(modin_groupby, pandas_groupby)
+        eval_general(modin_groupby, pandas_groupby, lambda df: df.cumsum(axis=0))
+        eval_general(modin_groupby, pandas_groupby, lambda df: df.cummax(axis=0))
+        eval_general(modin_groupby, pandas_groupby, lambda df: df.cummin(axis=0))
+        eval_general(modin_groupby, pandas_groupby, lambda df: df.cumprod(axis=0))
 
     eval_general(
         modin_groupby,
@@ -312,7 +326,7 @@ def test_simple_row_groupby(by, as_index):
         modin_df_almost_equals_pandas,
         is_default=True,
     )
-    eval_rank(modin_groupby, pandas_groupby)
+    eval_general(modin_groupby, pandas_groupby, lambda df: df.rank())
     eval_max(modin_groupby, pandas_groupby)
     eval_len(modin_groupby, pandas_groupby)
     eval_sum(modin_groupby, pandas_groupby)
@@ -332,7 +346,12 @@ def test_simple_row_groupby(by, as_index):
         # Pandas groupby.transform does not work correctly with NaN values in grouping columns. See Pandas bug 17093.
         transform_functions = [lambda df: df + 4, lambda df: -df - 10]
         for func in transform_functions:
-            eval_transform(modin_groupby, pandas_groupby, func)
+            eval_general(
+                modin_groupby,
+                pandas_groupby,
+                lambda df: df.transform(func),
+                check_exception_type=None,
+            )
 
     pipe_functions = [lambda dfgb: dfgb.sum()]
     for func in pipe_functions:
@@ -347,7 +366,9 @@ def test_simple_row_groupby(by, as_index):
     )
     eval_fillna(modin_groupby, pandas_groupby)
     eval_count(modin_groupby, pandas_groupby)
-    eval_size(modin_groupby, pandas_groupby)
+    eval_general(
+        modin_groupby, pandas_groupby, lambda df: df.size(), check_exception_type=None
+    )
     eval_general(modin_groupby, pandas_groupby, lambda df: df.tail(n), is_default=True)
     eval_quantile(modin_groupby, pandas_groupby)
     eval_general(modin_groupby, pandas_groupby, lambda df: df.take(), is_default=True)
@@ -471,7 +492,8 @@ def test_single_group_row_groupby():
     eval_groups(modin_groupby, pandas_groupby)
 
 
-def test_large_row_groupby():
+@pytest.mark.parametrize("is_by_category", [True, False])
+def test_large_row_groupby(is_by_category):
     pandas_df = pandas.DataFrame(
         np.random.randint(0, 8, size=(100, 4)), columns=list("ABCD")
     )
@@ -479,6 +501,10 @@ def test_large_row_groupby():
     modin_df = from_pandas(pandas_df)
 
     by = [str(i) for i in pandas_df["A"].tolist()]
+
+    if is_by_category:
+        by = pandas.Categorical(by)
+
     n = 4
 
     modin_groupby = modin_df.groupby(by=by)
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -381,8 +381,17 @@ def categories_equals(left, right):
 
 
 def df_categories_equals(df1, df2):
-    categories_columns = df1.select_dtypes(include="category").columns
+    if not hasattr(df1, "select_dtypes"):
+        if isinstance(df1, pandas.CategoricalDtype):
+            return categories_equals(df1, df2)
+        elif isinstance(getattr(df1, "dtype"), pandas.CategoricalDtype) and isinstance(
+            getattr(df1, "dtype"), pandas.CategoricalDtype
+        ):
+            return categories_equals(df1.dtype, df2.dtype)
+        else:
+            return True
 
+    categories_columns = df1.select_dtypes(include="category").columns
     for column in categories_columns:
         is_category_ordered = df1[column].dtype.ordered
         assert_categorical_equal(
@@ -558,17 +567,27 @@ def check_df_columns_have_nans(df, cols):
 
 
 def eval_general(
-    modin_df, pandas_df, operation, comparator=df_equals, __inplace__=False, **kwargs
+    modin_df,
+    pandas_df,
+    operation,
+    comparator=df_equals,
+    __inplace__=False,
+    check_exception_type=True,
+    **kwargs,
 ):
     md_kwargs, pd_kwargs = {}, {}
 
     def execute_callable(fn, inplace=False, md_kwargs={}, pd_kwargs={}):
         try:
             pd_result = fn(pandas_df, **pd_kwargs)
-        except Exception as e:
-            with pytest.raises(type(e)):
+        except Exception as pd_e:
+            if check_exception_type is None:
+                return None
+            with pytest.raises(Exception) as md_e:
                 # repr to force materialization
                 repr(fn(modin_df, **md_kwargs))
+            if check_exception_type:
+                assert isinstance(md_e.value, type(pd_e))
         else:
             md_result = fn(modin_df, **md_kwargs)
             return (md_result, pd_result) if not __inplace__ else (modin_df, pandas_df)