Repository: pandas-dev/pandas

Issue:
PERF: transform speedup
http://stackoverflow.com/questions/22072943/faster-way-to-transform-group-with-mean-value-in-pandas/22073449#22073449

I think this actually will work in a general case. but prob only really makes sense when you have a cythonized work function (or a ufunc) that is much faster than iteratively calling the groups.

```
np.random.seed(0)

N = 120000
N_TRANSITIONS = 1400

# generate groups
transition_points = np.random.permutation(np.arange(N))[:N_TRANSITIONS]
transition_points.sort()
transitions = np.zeros((N,), dtype=np.bool)
transitions[transition_points] = True
g = transitions.cumsum()

df = pd.DataFrame({ "signal" : np.random.rand(N)})
```

```
In [44]: grp = df["signal"].groupby(g)

In [45]: result2 = df["signal"].groupby(g).transform(np.mean)

In [47]: %timeit df["signal"].groupby(g).transform(np.mean)
1 loops, best of 3: 535 ms per loop
```

Using broadcasting

```
 In [43]: result = pd.concat([ Series([r]*len(grp.groups[i])) for i, r in enumerate(grp.mean().values) ],ignore_index=True)

In [42]: %timeit pd.concat([ Series([r]*len(grp.groups[i])) for i, r in enumerate(grp.mean().values) ],ignore_index=True)
10 loops, best of 3: 119 ms per loop

In [46]: result.equals(result2)
Out[46]: True
```

I think you might need to set the index of the returned on the broadcast result (it happens to work here because its a default index

```
result = pd.concat([ Series([r]*len(grp.groups[i])) for i, r in enumerate(grp.mean().values) ],ignore_index=True)
result.index = df.index
```

Final Result is best

```
pd.Series(np.repeat(grp.mean().values, grp.count().values))
```



Discussion:
That last one doesn't actually work for me for a familiar reason:

```
>>> pd.Series(np.repeat(grp.mean().values, grp.count().values))
Traceback (most recent call last):
  File "<ipython-input-12-c5646446d59d>", line 1, in <module>
    pd.Series(np.repeat(grp.mean().values, grp.count().values))
  File "/usr/local/lib/python2.7/dist-packages/numpy/core/fromnumeric.py", line 391, in repeat
    return repeat(repeats, axis)
TypeError: Cannot cast array data from dtype('int64') to dtype('int32') according to the rule 'safe'
```

with versions

```
>>> pd.show_versions()

INSTALLED VERSIONS
------------------
commit: None
python: 2.7.3.final.0
python-bits: 32
OS: Linux
OS-release: 3.2.0-58-generic-pae
machine: i686
processor: i686
byteorder: little
LC_ALL: None
LANG: en_CA.UTF-8

pandas: 0.13.1-328-gd0e2a9f
Cython: 0.20
numpy: 1.9.0.dev-3a2f048
scipy: 0.14.0.dev-432f16b
statsmodels: 0.6.0.dev-2bc4041
IPython: 1.2.0
sphinx: 1.2
patsy: 0.1.0
scikits.timeseries: None
dateutil: 1.5
pytz: 2013.9
bottleneck: 0.6.0
tables: 2.4.0
numexpr: 2.0.1
matplotlib: 1.4.x
openpyxl: 1.7.0
xlrd: 0.9.0
xlwt: 0.7.4
xlsxwriter: 0.5.2
lxml: 3.1.0
bs4: 4.1.0
html5lib: 0.95-dev
bq: None
apiclient: 1.2
rpy2: 2.3.9
sqlalchemy: 0.6.6
pymysql: None
psycopg2: None
```

I guess the numpy fixes from the other day didn't address this case.

and actually don't need to do that as this 

(and maybe already done in the `grp.grouper` object)

```
lens = dict([ (k,len(g)) for k,g in grp.grouper.groups.iteritems() ])
```

@dsm054 want to implement this?

@dsm054 interested?


Resolving patch:
diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -14,7 +14,7 @@
 from pandas.core.categorical import Categorical
 from pandas.core.frame import DataFrame
 from pandas.core.generic import NDFrame
-from pandas.core.index import Index, MultiIndex, _ensure_index
+from pandas.core.index import Index, MultiIndex, _ensure_index, _union_indexes
 from pandas.core.internals import BlockManager, make_block
 from pandas.core.series import Series
 from pandas.core.panel import Panel
@@ -425,7 +425,7 @@ def convert(key, s):
                 return Timestamp(key).asm8
             return key
 
-        sample = list(self.indices)[0]
+        sample = next(iter(self.indices))
         if isinstance(sample, tuple):
             if not isinstance(name, tuple):
                 raise ValueError("must supply a tuple to get_group with multiple grouping keys")
@@ -2193,33 +2193,37 @@ def transform(self, func, *args, **kwargs):
         -------
         transformed : Series
         """
-        result = self._selected_obj.copy()
-        if hasattr(result, 'values'):
-            result = result.values
-        dtype = result.dtype
+        dtype = self._selected_obj.dtype
 
         if isinstance(func, compat.string_types):
             wrapper = lambda x: getattr(x, func)(*args, **kwargs)
         else:
             wrapper = lambda x: func(x, *args, **kwargs)
 
-        for name, group in self:
+        result = self._selected_obj.values.copy()
+        for i, (name, group) in enumerate(self):
 
             object.__setattr__(group, 'name', name)
             res = wrapper(group)
+
             if hasattr(res, 'values'):
                 res = res.values
 
-            # need to do a safe put here, as the dtype may be different
-            # this needs to be an ndarray
-            result = Series(result)
-            result.iloc[self._get_index(name)] = res
-            result = result.values
+            # may need to astype
+            try:
+               common_type = np.common_type(np.array(res), result)
+               if common_type != result.dtype:
+                   result = result.astype(common_type)
+            except:
+                pass
+
+            indexer = self._get_index(name)
+            result[indexer] = res
 
-        # downcast if we can (and need)
         result = _possibly_downcast_to_dtype(result, dtype)
-        return self._selected_obj.__class__(result, index=self._selected_obj.index,
-                                  name=self._selected_obj.name)
+        return self._selected_obj.__class__(result,
+                                            index=self._selected_obj.index,
+                                            name=self._selected_obj.name)
 
     def filter(self, func, dropna=True, *args, **kwargs):
         """
diff --git a/vb_suite/groupby.py b/vb_suite/groupby.py
--- a/vb_suite/groupby.py
+++ b/vb_suite/groupby.py
@@ -376,3 +376,21 @@ def f(g):
 """
 
 groupby_transform = Benchmark("data.groupby(level='security_id').transform(f_fillna)", setup)
+
+setup = common_setup + """
+np.random.seed(0)
+
+N = 120000
+N_TRANSITIONS = 1400
+
+# generate groups
+transition_points = np.random.permutation(np.arange(N))[:N_TRANSITIONS]
+transition_points.sort()
+transitions = np.zeros((N,), dtype=np.bool)
+transitions[transition_points] = True
+g = transitions.cumsum()
+
+df = DataFrame({ 'signal' : np.random.rand(N)})
+"""
+
+groupby_transform2 = Benchmark("df['signal'].groupby(g).transform(np.mean)", setup)


Test patch:
diff --git a/pandas/tests/test_groupby.py b/pandas/tests/test_groupby.py
--- a/pandas/tests/test_groupby.py
+++ b/pandas/tests/test_groupby.py
@@ -126,8 +126,10 @@ def checkit(dtype):
             assert_series_equal(agged, grouped.mean())
             assert_series_equal(grouped.agg(np.sum), grouped.sum())
 
+            expected = grouped.apply(lambda x: x * x.sum())
             transformed = grouped.transform(lambda x: x * x.sum())
             self.assertEqual(transformed[7], 12)
+            assert_series_equal(transformed, expected)
 
             value_grouped = data.groupby(data)
             assert_series_equal(value_grouped.aggregate(np.mean), agged)