Repository: pandas-dev/pandas

Issue:
ENH/API: clarify groupby by to handle columns/index names
Referenced briefly in the OP at #3275

``` python
In [11]: idx = pd.MultiIndex.from_tuples([('a', 1), ('a', 2), ('a', 3), ('b', 1), ('b', 2), ('b', 3)])

In [12]: idx.names = ['outer', 'inner']

In [13]: df = pd.DataFrame({"A": np.arange(6), 'B': ['one', 'one', 'two', 'two', 'one', 'one']}, index=idx)
```

So the idea is to be able to call

``` python
df.groupby('B', level='inner')
```

instead of

``` python
In [15]: df.reset_index().groupby(['B', 'inner']).mean()
Out[15]: 
             A
B   inner     
one 1      0.0
    2      2.5
    3      5.0
two 1      3.0
    3      2.0

[5 rows x 1 columns]
```

Currently this raises `TypeError: 'numpy.ndarray' object is not callable`.  Mostly just syntactic sugar, but I've been having to do a lot of this lately and all the `reset_index`es are getting annoying. Thoughts?

Error when key-only Grouper is passed to groupby in a list
#### Overview

A `Grouper` object configured with only a `key` specification may be passed to `groupby` to group a DataFrame by a particular column. For example:

``` python
In [19]: import pandas as pd

In [20]: df = pd.DataFrame({'A': [0, 0, 0, 1, 1, 1], 
    ...:                    'B': [1, 1, 2, 2, 3, 3],
    ...:                    'C': [1, 2, 3, 4, 5, 6]})

In [26]: df.groupby(pd.Grouper(key='A')).count()
Out[26]: 
   B  C
A      
0  3  3
1  3  3
```

However, when this key-only `Grouper` is passed to `groupby` inside a list, a `TypeError` is thrown.  The following example is for a scalar list containing the same `Grouper` object as in the previous example. 

``` python
In [27]: df.groupby([pd.Grouper(key='A')]).count()
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-27-f4f86763ebfc> in <module>()
----> 1 df.groupby([pd.Grouper(key='A')]).count()

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/core/generic.py in groupby(self, by, axis, level, as_index, sort, group_keys, squeeze, **kwargs)
   3776         return groupby(self, by=by, axis=axis, level=level, as_index=as_index,
   3777                        sort=sort, group_keys=group_keys, squeeze=squeeze,
-> 3778                        **kwargs)
   3779 
   3780     def asfreq(self, freq, method=None, how=None, normalize=False):

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/core/groupby.py in groupby(obj, by, **kwds)
   1425         raise TypeError('invalid type: %s' % type(obj))
   1426 
-> 1427     return klass(obj, by, **kwds)
   1428 
   1429 

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/core/groupby.py in __init__(self, obj, keys, axis, level, grouper, exclusions, selection, as_index, sort, group_keys, squeeze, **kwargs)
    352                                                     level=level,
    353                                                     sort=sort,
--> 354                                                     mutated=self.mutated)
    355 
    356         self.obj = obj

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/core/groupby.py in _get_grouper(obj, key, axis, level, sort, mutated)
   2400                         sort=sort,
   2401                         in_axis=in_axis) \
-> 2402             if not isinstance(gpr, Grouping) else gpr
   2403 
   2404         groupings.append(ping)

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/core/groupby.py in __init__(self, index, grouper, obj, name, level, sort, in_axis)
   2197 
   2198                 # get the new grouper
-> 2199                 grouper = self.grouper._get_binner_for_grouping(self.obj)
   2200                 self.obj = self.grouper.obj
   2201                 self.grouper = grouper

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/core/groupby.py in _get_binner_for_grouping(self, obj)
    290         group_axis = obj._get_axis(self.axis)
    291         return Grouping(group_axis, None, obj=obj, name=self.key,
--> 292                         level=self.level, sort=self.sort, in_axis=False)
    293 
    294     @property

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/core/groupby.py in __init__(self, index, grouper, obj, name, level, sort, in_axis)
   2213                     t = self.name or str(type(self.grouper))
   2214                     raise ValueError("Grouper for '%s' not 1-dimensional" % t)
-> 2215                 self.grouper = self.index.map(self.grouper)
   2216                 if not (hasattr(self.grouper, "__len__") and
   2217                         len(self.grouper) == len(self.index)):

/Users/measejm1/anaconda/lib/python3.5/site-packages/pandas/indexes/base.py in map(self, mapper)
   2238         applied : array
   2239         """
-> 2240         return self._arrmap(self.values, mapper)
   2241 
   2242     def isin(self, values, level=None):

pandas/src/generated.pyx in pandas.algos.arrmap_int64 (pandas/algos.c:94003)()

TypeError: 'NoneType' object is not callable
```

The same error occurs for lists containing multiple key-only `Groupers`. e.g. `[pd.Grouper(key='A'), pd.Grouper(key='B')]`
#### Expected Output

``` python
In [27]: df.groupby([pd.Grouper(key='A')]).count()
Out[27]: 
   B  C
A      
0  3  3
1  3  3
```
#### Output of `pd.show_versions()`

<details>
## INSTALLED VERSIONS

commit: None
python: 3.5.2.final.0
python-bits: 64
OS: Darwin
OS-release: 15.6.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8

pandas: 0.18.1
nose: 1.3.7
pip: 8.1.2
setuptools: 23.0.0
Cython: 0.24
numpy: 1.11.1
scipy: 0.17.1
statsmodels: 0.6.1
xarray: None
IPython: 4.2.0
sphinx: 1.4.1
patsy: 0.4.1
dateutil: 2.5.3
pytz: 2016.4
blosc: 1.4.1
bottleneck: 1.1.0
tables: 3.2.2
numexpr: 2.6.0
matplotlib: 1.5.1
openpyxl: 2.3.2
xlrd: 1.0.0
xlwt: 1.1.2
xlsxwriter: 0.9.2
lxml: 3.6.0
bs4: 4.4.1
html5lib: 0.999
httplib2: None
apiclient: None
sqlalchemy: 1.0.13
pymysql: None
psycopg2: None
jinja2: 2.8
boto: 2.40.0
pandas_datareader: None
</details>



Discussion:
I take it the idea is to provide sugar for grouping on a combination of a column and and a multiindex level.
In that case, the example you provided works nicely, but:
- The group key spec is now smeared over multiple args (instead of just `by`). I think that's wrong.
- That approach doesn't handle a slightly more general case, grouping by a combination
  of multiple cols and levels in a certain order. That's a bad sign when it comes to API design.

How about resolving level names in the `by` handling code, and if there's a collision between
column names and multiindex names defaulting to the column (preserves existing code), but issuing
a warning. @jreback, does that sound right to you?

I don't think you even need these arguments, just an enhancement to figure out what the user wants. e.g

I see a label in the grouper, then follow a simple algo:
- index names (e.g. level)
- column name

That's just what I propose, except that variation breaks backwards-compat. 
Since `by` doesn't consider the index levels currently,  column names should have precedence IMO
to keep old code working. 
The warning should be there to alert the user that pandas is resolving an ambiguity by being opinionated. 

that's reasonable

@TomAugspurger so maybe let's change the name of this issue to something like 'clarify the grouper'?

so it deals nicely with index/columns names (and can raise/warn if their are duplicates, taking the columns in preference to the index names). If there STILL is ambiguity let's discuss (e.g.if you specify a label and it could possibly be misinterpreted somehow).

Just as an example:

``` python
In [4]: df=mkdf(4,2,r_idx_nlevels=2)
   ...: df.columns = ["foo","bar"]
   ...: df.index.names = ["baz","foo"]
   ...: df
Out[4]: 
                  foo   bar
baz     foo                
R_l0_g0 R_l1_g0  R0C0  R0C1
R_l0_g1 R_l1_g1  R1C0  R1C1
R_l0_g2 R_l1_g2  R2C0  R2C1
R_l0_g3 R_l1_g3  R3C0  R3C1

[4 rows x 2 columns]

In [5]: df.groupby('foo') # not ambiguous now, but would be
Out[5]: <pandas.core.groupby.DataFrameGroupBy object at 0x39e9d50>
```

Since groupy accepts an `axis` argument, I guess we need to consider the case for hirerchical columns as well.
Never used it myself nor can I reason out how to use it at the repl now that I tried, I expected 

``` python
df=pd.DataFrame([[1,1,3],['a','b','c']],index=['a','b'])
    ...: df.groupby(['a'],1).groups
KeyError: u'no item named a'
```

to be equivelent to:

``` python
df=pd.DataFrame([[1,1,3],['a','b','c']],index=['a','b'])
    ...: df.T.groupby(['a']).groups # notice transpose
{1: [0, 1], 3: [2]}
```

That all sounds reasonable. I can take a shot at implementing this in a few weeks. I'll probably have questions :)

The other bit is that `by` accepts (a list of) mapping functions, `dicts`, `Series`.  I don't think I've used either of these before.

@TomAugspurger I dont' think it accept a _list_ of mapping function / series, only a single mapper or series (otherwise should be an error)

From the docstring.

```
by : mapping function / list of functions, dict, Series, or tuple /
    list of column names.
    Called on each element of the object index to determine the groups.
    If a dict or Series is passed, the Series or dict VALUES will be
    used to determine the groups
```

I'll clarify that while I'm working on it.  I just have to figure out what that actually does first.

The docstring also says

```
>>> df.groupby.__doc__
...
Group **series** using mapper (dict or key function, apply given function\n       
```

Which is wrong.

@TomAugspurger I think this is worthwhile.....and prob not too complex....

?

@jreback @TomAugspurger I'm interested in implementing this.  Has anything changed since this discussion that I should be aware of?  I'll likely use Tom's old PR (#7033) as a starting point.

http://pandas.pydata.org/pandas-docs/stable/groupby.html#grouping-with-a-grouper-specification implements this

though you could make sugar for non colliding names which could be a name of an index / multi index

Ok, thanks.  So am I following that the way to accomplish the original example from this issue, without resetting the index, is the following?

``` python
In [75]: df.groupby(['B', pd.Grouper(level='inner')]).mean()
Out [75]: 
             A
B   inner     
one 1      0.0
    2      2.5
    3      5.0
two 1      3.0
    3      2.0
```

Should this approach also work when the frame has a singe named index? e.g.

``` python
In [76]: df2 = df.reset_index('outer')

In [77]: df2
Out [77]: 
      outer  A    B
inner              
1         a  0  one
2         a  1  one
3         a  2  two
1         b  3  two
2         b  4  one
3         b  5  one

In [79]: df2.groupby(['B', pd.Grouper(level='inner')]).mean()              
...
AttributeError: 'Int64Index' object has no attribute 'labels'
```

In this case I'm getting an Attribute Error (I'm on pandas 0.18.1 and happy to file a bug if this is one).

I would be interested in adding sugar in order to support

``` python
df.groupby(['B', 'inner']).mean()
```

and

``` python
df2.groupby(['B', 'inner']).mean()
```

where column names take precedence as discussed above. 

> Should this approach also work when the frame has a singe named index?

Yes, I think it should work. For example, it works when only specifying the index in this way:

```
In [40]: df2.groupby(level='inner').mean()
Out[40]: 
         A
inner     
1      1.5
2      2.5
3      3.5

In [42]: df2.groupby(pd.Grouper(level='inner')).mean()
Out[42]: 
         A
inner     
1      1.5
2      2.5
3      3.5

```

So using it in a list to group by multiple columns/indexes should also work. Do you want to open a separate bug report for this?

Yes, I'll open a separate bug report for this issue in a few hours.

This probably should work, thanks for the report!  But I'm curious, why are using a `Grouper` in this case instead of `df.groupby(['A'])`,  `df.groupby(['A','B'])`, etc?

Yeah, good question.  

I'm in the process of working on implementing #5677 which will allow the `A` in `df.groupby(['A'])` to refer to an index level in `df` named `A` if `df` has no column named `A`.  Per the conversation in #5677, If a frame has both a column named `A` and an index level named `A` then `df.groupby(['A'])` will group by the column (this maintains backwards compatibility).  I came across this bug in the process of writing tests to verify that this ambiguity is resolved properly.


Resolving patch:
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -3936,7 +3936,7 @@ def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True,
         Parameters
         ----------
         by : mapping function / list of functions, dict, Series, or tuple /
-            list of column names.
+            list of column names or index level names.
             Called on each element of the object index to determine the groups.
             If a dict or Series is passed, the Series or dict VALUES will be
             used to determine the groups
diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -2449,8 +2449,20 @@ def is_in_obj(gpr):
             exclusions.append(name)
 
         elif is_in_axis(gpr):  # df.groupby('name')
-            in_axis, name, gpr = True, gpr, obj[gpr]
-            exclusions.append(name)
+            if gpr in obj:
+                if gpr in obj.index.names:
+                    warnings.warn(
+                        ("'%s' is both a column name and an index level.\n"
+                         "Defaulting to column but "
+                         "this will raise an ambiguity error in a "
+                         "future version") % gpr,
+                        FutureWarning, stacklevel=2)
+                in_axis, name, gpr = True, gpr, obj[gpr]
+                exclusions.append(name)
+            elif gpr in obj.index.names:
+                in_axis, name, level, gpr = False, None, gpr, None
+            else:
+                raise KeyError(gpr)
         elif isinstance(gpr, Grouper) and gpr.key is not None:
             # Add key to exclusions
             exclusions.append(gpr.key)


Test patch:
diff --git a/pandas/tests/test_groupby.py b/pandas/tests/test_groupby.py
--- a/pandas/tests/test_groupby.py
+++ b/pandas/tests/test_groupby.py
@@ -521,6 +521,158 @@ def test_grouper_column_and_index(self):
         expected = df_single.reset_index().groupby(['inner', 'B']).mean()
         assert_frame_equal(result, expected)
 
+    def test_grouper_index_level_as_string(self):
+        # GH 5677, allow strings passed as the `by` parameter to reference
+        # columns or index levels
+
+        idx = pd.MultiIndex.from_tuples([('a', 1), ('a', 2), ('a', 3),
+                                         ('b', 1), ('b', 2), ('b', 3)])
+        idx.names = ['outer', 'inner']
+        df_multi = pd.DataFrame({"A": np.arange(6),
+                                 'B': ['one', 'one', 'two',
+                                       'two', 'one', 'one']},
+                                index=idx)
+
+        df_single = df_multi.reset_index('outer')
+
+        # Column and Index on MultiIndex
+        result = df_multi.groupby(['B', 'inner']).mean()
+        expected = df_multi.groupby(['B', pd.Grouper(level='inner')]).mean()
+        assert_frame_equal(result, expected)
+
+        # Index and Column on MultiIndex
+        result = df_multi.groupby(['inner', 'B']).mean()
+        expected = df_multi.groupby([pd.Grouper(level='inner'), 'B']).mean()
+        assert_frame_equal(result, expected)
+
+        # Column and Index on single Index
+        result = df_single.groupby(['B', 'inner']).mean()
+        expected = df_single.groupby(['B', pd.Grouper(level='inner')]).mean()
+        assert_frame_equal(result, expected)
+
+        # Index and Column on single Index
+        result = df_single.groupby(['inner', 'B']).mean()
+        expected = df_single.groupby([pd.Grouper(level='inner'), 'B']).mean()
+        assert_frame_equal(result, expected)
+
+        # Single element list of Index on MultiIndex
+        result = df_multi.groupby(['inner']).mean()
+        expected = df_multi.groupby(pd.Grouper(level='inner')).mean()
+        assert_frame_equal(result, expected)
+
+        # Single element list of Index on single Index
+        result = df_single.groupby(['inner']).mean()
+        expected = df_single.groupby(pd.Grouper(level='inner')).mean()
+        assert_frame_equal(result, expected)
+
+        # Index on MultiIndex
+        result = df_multi.groupby('inner').mean()
+        expected = df_multi.groupby(pd.Grouper(level='inner')).mean()
+        assert_frame_equal(result, expected)
+
+        # Index on single Index
+        result = df_single.groupby('inner').mean()
+        expected = df_single.groupby(pd.Grouper(level='inner')).mean()
+        assert_frame_equal(result, expected)
+
+    def test_grouper_column_index_level_precedence(self):
+        # GH 5677, when a string passed as the `by` parameter
+        # matches a column and an index level the column takes
+        # precedence
+
+        idx = pd.MultiIndex.from_tuples([('a', 1), ('a', 2), ('a', 3),
+                                         ('b', 1), ('b', 2), ('b', 3)])
+        idx.names = ['outer', 'inner']
+        df_multi_both = pd.DataFrame({"A": np.arange(6),
+                                      'B': ['one', 'one', 'two',
+                                            'two', 'one', 'one'],
+                                      'inner': [1, 1, 1, 1, 1, 1]},
+                                     index=idx)
+
+        df_single_both = df_multi_both.reset_index('outer')
+
+        # Group MultiIndex by single key
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_multi_both.groupby('inner').mean()
+
+        expected = df_multi_both.groupby([pd.Grouper(key='inner')]).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_multi_both.groupby(pd.Grouper(level='inner')).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
+        # Group single Index by single key
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_single_both.groupby('inner').mean()
+
+        expected = df_single_both.groupby([pd.Grouper(key='inner')]).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_single_both.groupby(pd.Grouper(level='inner')).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
+        # Group MultiIndex by single key list
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_multi_both.groupby(['inner']).mean()
+
+        expected = df_multi_both.groupby([pd.Grouper(key='inner')]).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_multi_both.groupby(pd.Grouper(level='inner')).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
+        # Group single Index by single key list
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_single_both.groupby(['inner']).mean()
+
+        expected = df_single_both.groupby([pd.Grouper(key='inner')]).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_single_both.groupby(pd.Grouper(level='inner')).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
+        # Group MultiIndex by two keys (1)
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_multi_both.groupby(['B', 'inner']).mean()
+
+        expected = df_multi_both.groupby(['B',
+                                          pd.Grouper(key='inner')]).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_multi_both.groupby(['B',
+                                              pd.Grouper(level='inner')
+                                              ]).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
+        # Group MultiIndex by two keys (2)
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_multi_both.groupby(['inner', 'B']).mean()
+
+        expected = df_multi_both.groupby([pd.Grouper(key='inner'),
+                                          'B']).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_multi_both.groupby([pd.Grouper(level='inner'),
+                                              'B']).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
+        # Group single Index by two keys (1)
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_single_both.groupby(['B', 'inner']).mean()
+
+        expected = df_single_both.groupby(['B',
+                                           pd.Grouper(key='inner')]).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_single_both.groupby(['B',
+                                               pd.Grouper(level='inner')
+                                               ]).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
+        # Group single Index by two keys (2)
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df_single_both.groupby(['inner', 'B']).mean()
+
+        expected = df_single_both.groupby([pd.Grouper(key='inner'),
+                                           'B']).mean()
+        assert_frame_equal(result, expected)
+        not_expected = df_single_both.groupby([pd.Grouper(level='inner'),
+                                               'B']).mean()
+        self.assertFalse(result.index.equals(not_expected.index))
+
     def test_grouper_getting_correct_binner(self):
 
         # GH 10063