Repository: pandas-dev/pandas

Issue:
Groupby filter doesn't work with repeated index
Example:

```
In [1]: data = pandas.DataFrame(
    {'pid' : [1,1,1,2,2,3,3,3],
     'tag' : [23,45,62,24,45,34,25,62],
     }, index=[0] * 8)

In [2]: g = data.groupby('tag')

In [3]: g.filter(lambda x: len(x) > 1)
Exception: Reindexing only valid with uniquely valued Index objects
```

cc #3680

_(I had thought there was also a similar behaviour with transform, but that seems ok.)_



Discussion:
cc @danielballan while you're looking at the related issue :)

There is a problem with transform also. There are four cases to consider: filtering and transforming both DataFrame- and SeriesGroupBy. Transforming a DataFrameGroupBy works.

```
In [28]: df_groupby = data.groupby('tag') # a DataFrameGroupBy object

In [29]: df_groupby.transform(len)
Out[29]: 
   pid
0    1
0    1
0    1
0    1
0    2
0    2
0    2
0    2
```

But the other three cases do not.

```
In [30]: df_groupby.filter(lambda x: len(x) > 1)
InvalidIndexError: Reindexing only valid with uniquely valued Index objects

In [31]: ser_groupby = data['pid'].groupby(data['tag'])

In [32]: ser_groupby.transform(len)
InvalidIndexError: Reindexing only valid with uniquely valued Index objects

In [33]: ser_groupby.filter(lambda x: len(x) > 1)
InvalidIndexError: Reindexing only valid with uniquely valued Index objects
```

Can anyone explain why this is not a valid use of `get_indexer`?

```
DataFrame([1,2,3], [0, 0, 1]).index.get_indexer([0])
---------------------------------------------------------------------------
InvalidIndexError                         Traceback (most recent call last)
<ipython-input-42-ad97732098af> in <module>()
----> 1 DataFrame([1,2,3], [0, 0, 1]).index.get_indexer([0])

/home/dallan/pandas-danielballan/pandas/core/index.py in get_indexer(self, target, method, limit)
   1112 
   1113         if not self.is_unique:
-> 1114             raise InvalidIndexError('Reindexing only valid with uniquely'
   1115                                     ' valued Index objects')
   1116 

InvalidIndexError: Reindexing only valid with uniquely valued Index objects
```

I would expect the output `array([0, 1])`.

You need to know if they are unique or not, `index.is_unique`, then you can use `get_indexer`, else you can use `get_indexer_non_unique` (they are separate because you don't always want/need to do this generally), and finding non-unique indexers is somewhat more expensive perf wise (though not much)

```
In [6]: df.index.get_indexer_non_unique([0])
Out[6]: (Int64Index([0, 1], dtype='int64'), array([], dtype=int64))
```

Ah, thanks.

For reference, get_indexer_non_unique was not the key here. See commit notes on #5375 .


Resolving patch:
diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -566,6 +566,22 @@ def _concat_objects(self, keys, values, not_indexed_same=False):
 
         return result
 
+    def _apply_filter(self, indices, dropna):
+        if len(indices) == 0:
+            indices = []
+        else:
+            indices = np.sort(np.concatenate(indices))
+        if dropna:
+            filtered = self.obj.take(indices)
+        else:
+            mask = np.empty(len(self.obj.index), dtype=bool)
+            mask.fill(False)
+            mask[indices.astype(int)] = True
+            # mask fails to broadcast when passed to where; broadcast manually.
+            mask = np.tile(mask, list(self.obj.shape[1:]) + [1]).T
+            filtered = self.obj.where(mask) # Fill with NaNs.
+        return filtered
+
 
 @Appender(GroupBy.__doc__)
 def groupby(obj, by, **kwds):
@@ -1585,14 +1601,13 @@ def transform(self, func, *args, **kwargs):
             group = com.ensure_float(group)
             object.__setattr__(group, 'name', name)
             res = wrapper(group)
-            indexer = self.obj.index.get_indexer(group.index)
             if hasattr(res,'values'):
                 res = res.values
 
             # need to do a safe put here, as the dtype may be different
             # this needs to be an ndarray
             result = Series(result)
-            result.loc[indexer] = res
+            result.iloc[self.indices[name]] = res
             result = result.values
 
         # downcast if we can (and need)
@@ -1630,22 +1645,15 @@ def true_and_notnull(x, *args, **kwargs):
             return b and notnull(b)
 
         try:
-            indexers = [self.obj.index.get_indexer(group.index) \
-                        if true_and_notnull(group) else [] \
-                        for _ , group in self]
+            indices = [self.indices[name] if true_and_notnull(group) else [] 
+                        for name, group in self]
         except ValueError:
             raise TypeError("the filter must return a boolean result")
         except TypeError:
             raise TypeError("the filter must return a boolean result")
 
-        if len(indexers) == 0:
-            filtered = self.obj.take([]) # because np.concatenate would fail
-        else:
-            filtered = self.obj.take(np.sort(np.concatenate(indexers)))
-        if dropna:
-            return filtered
-        else:
-            return filtered.reindex(self.obj.index) # Fill with NaNs.
+        filtered = self._apply_filter(indices, dropna)
+        return filtered
 
 
 class NDFrameGroupBy(GroupBy):
@@ -2125,7 +2133,7 @@ def filter(self, func, dropna=True, *args, **kwargs):
         """
         from pandas.tools.merge import concat
 
-        indexers = []
+        indices = []
 
         obj = self._obj_with_exclusions
         gen = self.grouper.get_iterator(obj, axis=self.axis)
@@ -2146,31 +2154,25 @@ def filter(self, func, dropna=True, *args, **kwargs):
             else:
                 res = path(group)
 
-            def add_indexer():
-                indexers.append(self.obj.index.get_indexer(group.index))
+            def add_indices():
+                indices.append(self.indices[name])
 
             # interpret the result of the filter
             if isinstance(res,(bool,np.bool_)):
                 if res:
-                    add_indexer()
+                    add_indices()
             else:
                 if getattr(res,'ndim',None) == 1:
                     val = res.ravel()[0]
                     if val and notnull(val):
-                        add_indexer()
+                        add_indices()
                 else:
 
                     # in theory you could do .all() on the boolean result ?
                     raise TypeError("the filter must return a boolean result")
 
-        if len(indexers) == 0:
-            filtered = self.obj.take([]) # because np.concatenate would fail
-        else:
-            filtered = self.obj.take(np.sort(np.concatenate(indexers)))
-        if dropna:
-            return filtered
-        else:
-            return filtered.reindex(self.obj.index) # Fill with NaNs.
+        filtered = self._apply_filter(indices, dropna)
+        return filtered
 
 
 class DataFrameGroupBy(NDFrameGroupBy):


Test patch:
diff --git a/pandas/tests/test_groupby.py b/pandas/tests/test_groupby.py
--- a/pandas/tests/test_groupby.py
+++ b/pandas/tests/test_groupby.py
@@ -7,8 +7,8 @@
 from datetime import datetime
 from numpy import nan
 
-from pandas import bdate_range
-from pandas.core.index import Index, MultiIndex
+from pandas import bdate_range, Timestamp
+from pandas.core.index import Index, MultiIndex, Int64Index
 from pandas.core.common import rands
 from pandas.core.api import Categorical, DataFrame
 from pandas.core.groupby import SpecificationError, DataError
@@ -2801,6 +2801,277 @@ def test_filter_maintains_ordering(self):
         expected = s.iloc[[1, 2, 4, 7]]
         assert_series_equal(actual, expected)
 
+    def test_filter_and_transform_with_non_unique_int_index(self):
+        # GH4620
+        index = [1, 1, 1, 2, 1, 1, 0, 1]
+        df = DataFrame({'pid' : [1,1,1,2,2,3,3,3], 
+                       'tag' : [23,45,62,24,45,34,25,62]}, index=index)
+        grouped_df = df.groupby('tag')
+        ser = df['pid']
+        grouped_ser = ser.groupby(df['tag'])
+        expected_indexes = [1, 2, 4, 7]
+
+        # Filter DataFrame
+        actual = grouped_df.filter(lambda x: len(x) > 1)
+        expected = df.iloc[expected_indexes]
+        assert_frame_equal(actual, expected)
+
+        actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False)
+        expected = df.copy()
+        expected.iloc[[0, 3, 5, 6]] = np.nan
+        assert_frame_equal(actual, expected)
+
+        # Filter Series
+        actual = grouped_ser.filter(lambda x: len(x) > 1)
+        expected = ser.take(expected_indexes)
+        assert_series_equal(actual, expected)
+
+        actual = grouped_ser.filter(lambda x: len(x) > 1, dropna=False)
+        NA = np.nan
+        expected = Series([NA,1,1,NA,2,NA,NA,3], index, name='pid')
+        # ^ made manually because this can get confusing!
+        assert_series_equal(actual, expected)
+
+        # Transform Series 
+        actual = grouped_ser.transform(len)
+        expected = Series([1, 2, 2, 1, 2, 1, 1, 2], index)
+        assert_series_equal(actual, expected)
+
+        # Transform (a column from) DataFrameGroupBy
+        actual = grouped_df.pid.transform(len)
+        assert_series_equal(actual, expected)
+
+    def test_filter_and_transform_with_multiple_non_unique_int_index(self):
+        # GH4620
+        index = [1, 1, 1, 2, 0, 0, 0, 1]
+        df = DataFrame({'pid' : [1,1,1,2,2,3,3,3],
+                       'tag' : [23,45,62,24,45,34,25,62]}, index=index)
+        grouped_df = df.groupby('tag')
+        ser = df['pid']
+        grouped_ser = ser.groupby(df['tag'])
+        expected_indexes = [1, 2, 4, 7]
+
+        # Filter DataFrame
+        actual = grouped_df.filter(lambda x: len(x) > 1)
+        expected = df.iloc[expected_indexes]
+        assert_frame_equal(actual, expected)
+
+        actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False)
+        expected = df.copy()
+        expected.iloc[[0, 3, 5, 6]] = np.nan
+        assert_frame_equal(actual, expected)
+
+        # Filter Series
+        actual = grouped_ser.filter(lambda x: len(x) > 1)
+        expected = ser.take(expected_indexes)
+        assert_series_equal(actual, expected)
+
+        actual = grouped_ser.filter(lambda x: len(x) > 1, dropna=False)
+        NA = np.nan
+        expected = Series([NA,1,1,NA,2,NA,NA,3], index, name='pid')
+        # ^ made manually because this can get confusing!
+        assert_series_equal(actual, expected)
+
+        # Transform Series
+        actual = grouped_ser.transform(len)
+        expected = Series([1, 2, 2, 1, 2, 1, 1, 2], index)
+        assert_series_equal(actual, expected)
+
+        # Transform (a column from) DataFrameGroupBy
+        actual = grouped_df.pid.transform(len)
+        assert_series_equal(actual, expected)
+
+    def test_filter_and_transform_with_non_unique_float_index(self):
+        # GH4620
+        index = np.array([1, 1, 1, 2, 1, 1, 0, 1], dtype=float)
+        df = DataFrame({'pid' : [1,1,1,2,2,3,3,3],
+                       'tag' : [23,45,62,24,45,34,25,62]}, index=index)
+        grouped_df = df.groupby('tag')
+        ser = df['pid']
+        grouped_ser = ser.groupby(df['tag'])
+        expected_indexes = [1, 2, 4, 7]
+
+        # Filter DataFrame
+        actual = grouped_df.filter(lambda x: len(x) > 1)
+        expected = df.iloc[expected_indexes]
+        assert_frame_equal(actual, expected)
+
+        actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False)
+        expected = df.copy()
+        expected.iloc[[0, 3, 5, 6]] = np.nan
+        assert_frame_equal(actual, expected)
+
+        # Filter Series
+        actual = grouped_ser.filter(lambda x: len(x) > 1)
+        expected = ser.take(expected_indexes)
+        assert_series_equal(actual, expected)
+
+        actual = grouped_ser.filter(lambda x: len(x) > 1, dropna=False)
+        NA = np.nan
+        expected = Series([NA,1,1,NA,2,NA,NA,3], index, name='pid')
+        # ^ made manually because this can get confusing!
+        assert_series_equal(actual, expected)
+
+        # Transform Series
+        actual = grouped_ser.transform(len)
+        expected = Series([1, 2, 2, 1, 2, 1, 1, 2], index)
+        assert_series_equal(actual, expected)
+
+        # Transform (a column from) DataFrameGroupBy
+        actual = grouped_df.pid.transform(len)
+        assert_series_equal(actual, expected)
+
+    def test_filter_and_transform_with_non_unique_float_index(self):
+        # GH4620
+        index = np.array([1, 1, 1, 2, 0, 0, 0, 1], dtype=float)
+        df = DataFrame({'pid' : [1,1,1,2,2,3,3,3],
+                       'tag' : [23,45,62,24,45,34,25,62]}, index=index)
+        grouped_df = df.groupby('tag')
+        ser = df['pid']
+        grouped_ser = ser.groupby(df['tag'])
+        expected_indexes = [1, 2, 4, 7]
+
+        # Filter DataFrame
+        actual = grouped_df.filter(lambda x: len(x) > 1)
+        expected = df.iloc[expected_indexes]
+        assert_frame_equal(actual, expected)
+
+        actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False)
+        expected = df.copy()
+        expected.iloc[[0, 3, 5, 6]] = np.nan
+        assert_frame_equal(actual, expected)
+
+        # Filter Series
+        actual = grouped_ser.filter(lambda x: len(x) > 1)
+        expected = ser.take(expected_indexes)
+        assert_series_equal(actual, expected)
+
+        actual = grouped_ser.filter(lambda x: len(x) > 1, dropna=False)
+        NA = np.nan
+        expected = Series([NA,1,1,NA,2,NA,NA,3], index, name='pid')
+        # ^ made manually because this can get confusing!
+        assert_series_equal(actual, expected)
+
+        # Transform Series
+        actual = grouped_ser.transform(len)
+        expected = Series([1, 2, 2, 1, 2, 1, 1, 2], index)
+        assert_series_equal(actual, expected)
+
+        # Transform (a column from) DataFrameGroupBy
+        actual = grouped_df.pid.transform(len)
+        assert_series_equal(actual, expected)
+
+    def test_filter_and_transform_with_non_unique_timestamp_index(self):
+        # GH4620
+        t0 = Timestamp('2013-09-30 00:05:00')
+        t1 = Timestamp('2013-10-30 00:05:00')
+        t2 = Timestamp('2013-11-30 00:05:00')
+        index = [t1, t1, t1, t2, t1, t1, t0, t1]
+        df = DataFrame({'pid' : [1,1,1,2,2,3,3,3],
+                       'tag' : [23,45,62,24,45,34,25,62]}, index=index)
+        grouped_df = df.groupby('tag')
+        ser = df['pid']
+        grouped_ser = ser.groupby(df['tag'])
+        expected_indexes = [1, 2, 4, 7]
+
+        # Filter DataFrame
+        actual = grouped_df.filter(lambda x: len(x) > 1)
+        expected = df.iloc[expected_indexes]
+        assert_frame_equal(actual, expected)
+
+        actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False)
+        expected = df.copy()
+        expected.iloc[[0, 3, 5, 6]] = np.nan
+        assert_frame_equal(actual, expected)
+
+        # Filter Series
+        actual = grouped_ser.filter(lambda x: len(x) > 1)
+        expected = ser.take(expected_indexes)
+        assert_series_equal(actual, expected)
+
+        actual = grouped_ser.filter(lambda x: len(x) > 1, dropna=False)
+        NA = np.nan
+        expected = Series([NA,1,1,NA,2,NA,NA,3], index, name='pid')
+        # ^ made manually because this can get confusing!
+        assert_series_equal(actual, expected)
+
+        # Transform Series
+        actual = grouped_ser.transform(len)
+        expected = Series([1, 2, 2, 1, 2, 1, 1, 2], index)
+        assert_series_equal(actual, expected)
+
+        # Transform (a column from) DataFrameGroupBy
+        actual = grouped_df.pid.transform(len)
+        assert_series_equal(actual, expected)
+
+    def test_filter_and_transform_with_non_unique_string_index(self):
+        # GH4620
+        index = list('bbbcbbab')
+        df = DataFrame({'pid' : [1,1,1,2,2,3,3,3],
+                       'tag' : [23,45,62,24,45,34,25,62]}, index=index)
+        grouped_df = df.groupby('tag')
+        ser = df['pid']
+        grouped_ser = ser.groupby(df['tag'])
+        expected_indexes = [1, 2, 4, 7]
+
+        # Filter DataFrame
+        actual = grouped_df.filter(lambda x: len(x) > 1)
+        expected = df.iloc[expected_indexes]
+        assert_frame_equal(actual, expected)
+
+        actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False)
+        expected = df.copy()
+        expected.iloc[[0, 3, 5, 6]] = np.nan
+        assert_frame_equal(actual, expected)
+
+        # Filter Series
+        actual = grouped_ser.filter(lambda x: len(x) > 1)
+        expected = ser.take(expected_indexes)
+        assert_series_equal(actual, expected)
+
+        actual = grouped_ser.filter(lambda x: len(x) > 1, dropna=False)
+        NA = np.nan
+        expected = Series([NA,1,1,NA,2,NA,NA,3], index, name='pid')
+        # ^ made manually because this can get confusing!
+        assert_series_equal(actual, expected)
+
+        # Transform Series
+        actual = grouped_ser.transform(len)
+        expected = Series([1, 2, 2, 1, 2, 1, 1, 2], index)
+        assert_series_equal(actual, expected)
+
+        # Transform (a column from) DataFrameGroupBy
+        actual = grouped_df.pid.transform(len)
+        assert_series_equal(actual, expected)
+
+    def test_index_label_overlaps_location(self):
+        # checking we don't have any label/location confusion in the
+        # the wake of GH5375
+        df = DataFrame(list('ABCDE'), index=[2, 0, 2, 1, 1])
+        g = df.groupby(list('ababb'))
+        actual = g.filter(lambda x: len(x) > 2)
+        expected = df.iloc[[1, 3, 4]]
+        assert_frame_equal(actual, expected)
+
+        ser = df[0]
+        g = ser.groupby(list('ababb'))
+        actual = g.filter(lambda x: len(x) > 2)
+        expected = ser.take([1, 3, 4])
+        assert_series_equal(actual, expected)
+
+        # ... and again, with a generic Index of floats
+        df.index = df.index.astype(float)
+        g = df.groupby(list('ababb'))
+        actual = g.filter(lambda x: len(x) > 2)
+        expected = df.iloc[[1, 3, 4]]
+        assert_frame_equal(actual, expected)
+
+        ser = df[0]
+        g = ser.groupby(list('ababb'))
+        actual = g.filter(lambda x: len(x) > 2)
+        expected = ser.take([1, 3, 4])
+        assert_series_equal(actual, expected)
+
     def test_groupby_whitelist(self):
         from string import ascii_lowercase
         letters = np.array(list(ascii_lowercase))