Repository: pandas-dev/pandas

Issue:
GroupBy.transform bug/API change
hi folks,

A _Python for Data Analysis_ reader noted the following issue with recent versions of pandas (as of 1 year ago):

```
import pandas as pd
from pandas import DataFrame
import numpy as np

def demean(arr):
    return arr - arr.mean()

people = DataFrame(np.random.randn(5, 5),
columns=['a', 'b', 'c', 'd', 'e'],
index=['Joe', 'Steve', 'Wes', 'Jim', 'Travis'])
key = ['one', 'two', 'one', 'two', 'one']
```

on pandas 0.14.1:

```
In [14]: people.groupby(key).transform(demean).groupby(key).mean()
Out[14]: 
            a         b         c         d         e
one -0.228006  0.246737  0.201117  0.250544  0.273858
two  0.342009 -0.370106 -0.301676 -0.375816 -0.410788
```

on the other hand:

```
In [15]: people.groupby(key).apply(demean).groupby(key).mean()
Out[15]: 
                a             b             c             d             e
one -3.700743e-17  7.401487e-17 -7.401487e-17  7.401487e-17  0.000000e+00
two  0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00  5.551115e-17
```

Looks like `transform` has undergone some work in recent times; any ideas? I need to look at the book text and see if I can triage by replacing `transform` with `apply`. At this point `transform` feels a little bit anachronistic. 



Discussion:
We're not preserving the index order when we return from transform, so `key` doesn't mean the same thing in each case: 

```
In [119]: df = pd.DataFrame(np.arange(6).reshape(3,2), columns=["a","b"], index=[0,2,1])

In [120]: key = [0,0,1]

In [121]: df
Out[121]: 
   a  b
0  0  1
2  2  3
1  4  5

In [122]: df.groupby(key).transform(lambda x: x-x.mean())
Out[122]: 
   a  b
0 -1 -1
1  0  0
2  1  1

In [123]: df.groupby(key).transform(lambda x: x-x.mean()).groupby(key).mean()
Out[123]: 
     a    b
0 -0.5 -0.5
1  1.0  1.0

In [124]: df.sort_index().groupby(key).transform(lambda x: x-x.mean()).groupby(key).mean()
Out[124]: 
   a  b
0  0  0
1  0  0
```

Should be straightforward enough to fix.

A reader noted that this works:

```
mapc = {'joe':'one', 'steve':'two', 'wes':'one', 'jim':'two', 'travis':'one'}

and now the the following produces zero mean:

>>> demeaned = p.groupby(mapc).transform(demean)
>>> demeaned.groupby(mapc).mean()
a b c d e
one 7.401487e-17 0 3.700743e-17 3.700743e-17 -4.625929e-17
two 0.000000e+00 0 -1.387779e-17 5.551115e-17 0.000000e+00
```

If I'm right about the problem, then it makes sense that using a dictionary should work, because the dictionary doesn't depend upon the order of the indices to establish the group mapping.


Resolving patch:
diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -475,6 +475,24 @@ def _set_selection_from_grouper(self):
             if len(groupers):
                 self._group_selection = (ax-Index(groupers)).tolist()
 
+    def _set_result_index_ordered(self, result):
+        # set the result index on the passed values object
+        # return the new object
+        # related 8046
+
+        # the values/counts are repeated according to the group index
+        indices = self.indices
+
+        # shortcut of we have an already ordered grouper
+
+        if not Index(self.grouper.group_info[0]).is_monotonic:
+            index = Index(np.concatenate([ indices[v] for v in self.grouper.result_index ]))
+            result.index = index
+            result = result.sort_index()
+
+        result.index = self.obj.index
+        return result
+
     def _local_dir(self):
         return sorted(set(self.obj._local_dir() + list(self._apply_whitelist)))
 
@@ -2087,7 +2105,6 @@ def _convert_grouper(axis, grouper):
     else:
         return grouper
 
-
 class SeriesGroupBy(GroupBy):
     _apply_whitelist = _series_apply_whitelist
 
@@ -2319,18 +2336,7 @@ def _transform_fast(self, func):
         counts = self.count().values
         values = np.repeat(values, com._ensure_platform_int(counts))
 
-        # the values/counts are repeated according to the group index
-        indices = self.indices
-
-        # shortcut of we have an already ordered grouper
-        if Index(self.grouper.group_info[0]).is_monotonic:
-            result = Series(values, index=self.obj.index)
-        else:
-            index = Index(np.concatenate([ indices[v] for v in self.grouper.result_index ]))
-            result = Series(values, index=index).sort_index()
-            result.index = self.obj.index
-
-        return result
+        return self._set_result_index_ordered(Series(values))
 
     def filter(self, func, dropna=True, *args, **kwargs):
         """
@@ -2842,8 +2848,7 @@ def _transform_general(self, func, *args, **kwargs):
         concat_index = obj.columns if self.axis == 0 else obj.index
         concatenated = concat(applied, join_axes=[concat_index],
                               axis=self.axis, verify_integrity=False)
-        concatenated.sort_index(inplace=True)
-        return concatenated
+        return self._set_result_index_ordered(concatenated)
 
     def transform(self, func, *args, **kwargs):
         """


Test patch:
diff --git a/pandas/tests/test_groupby.py b/pandas/tests/test_groupby.py
--- a/pandas/tests/test_groupby.py
+++ b/pandas/tests/test_groupby.py
@@ -796,6 +796,26 @@ def test_transform(self):
         transformed = grouped.transform(lambda x: x * x.sum())
         self.assertEqual(transformed[7], 12)
 
+        # GH 8046
+        # make sure that we preserve the input order
+
+        df = DataFrame(np.arange(6,dtype='int64').reshape(3,2), columns=["a","b"], index=[0,2,1])
+        key = [0,0,1]
+        expected = df.sort_index().groupby(key).transform(lambda x: x-x.mean()).groupby(key).mean()
+        result = df.groupby(key).transform(lambda x: x-x.mean()).groupby(key).mean()
+        assert_frame_equal(result, expected)
+
+        def demean(arr):
+            return arr - arr.mean()
+
+        people = DataFrame(np.random.randn(5, 5),
+                           columns=['a', 'b', 'c', 'd', 'e'],
+                           index=['Joe', 'Steve', 'Wes', 'Jim', 'Travis'])
+        key = ['one', 'two', 'one', 'two', 'one']
+        result = people.groupby(key).transform(demean).groupby(key).mean()
+        expected = people.groupby(key).apply(demean).groupby(key).mean()
+        assert_frame_equal(result, expected)
+
     def test_transform_fast(self):
 
         df = DataFrame( { 'id' : np.arange( 100000 ) / 3,
@@ -2924,7 +2944,7 @@ def __call__(self, x):
                            lambda x: sum(x),
                            lambda x: x.sum(),
                            partial(sum), fn_class()]
-        
+
         expected = df.groupby("foo").agg(sum)
         for ecall in equiv_callables:
             result = df.groupby('foo').agg(ecall)