Repository: pandas-dev/pandas

Issue:
PERF: cythonize groupby-rank
This dispatches to each group individually. Better to have a combined group_rank to do this. It is a bit of code and ideally would share some with the actual rank algos.

```
In [7]: ngroups = 1000

In [8]: N = 100000

In [9]: np.random.seed(1234)

In [10]: df = DataFrame({'key': np.random.randint(0, ngroups, size=N), 'value': np.arange(N)})

In [11]: %timeit df.groupby('key').rank()
1 loop, best of 3: 392 ms per loop

# comparision with group_shift_indexer, a transforming operator
In [13]: %timeit df.groupby('key').shift()
100 loops, best of 3: 3.15 ms per loop
```



Discussion:
I can take a look at this. Any tips on what methods to explore? I was thinking of adding a method to the GroupBy class similar to the others for rank and was looking at the rank method in algos. 

It wasn't immediately clear to me the best way to knit that all together so figured I'd get your thoughts if you have any
yeah u can make a separate rank routine which takes a group indexer; eg copy something like group_last and integrate with rank 

Resolving patch:
diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -994,7 +994,7 @@ def _transform_should_cast(self, func_nm):
         return (self.size().fillna(0) > 0).any() and (func_nm not in
                                                       _cython_cast_blacklist)
 
-    def _cython_transform(self, how, numeric_only=True):
+    def _cython_transform(self, how, numeric_only=True, **kwargs):
         output = collections.OrderedDict()
         for name, obj in self._iterate_slices():
             is_numeric = is_numeric_dtype(obj.dtype)
@@ -1002,12 +1002,16 @@ def _cython_transform(self, how, numeric_only=True):
                 continue
 
             try:
-                result, names = self.grouper.transform(obj.values, how)
+                result, names = self.grouper.transform(obj.values, how,
+                                                       **kwargs)
             except NotImplementedError:
                 continue
             except AssertionError as e:
                 raise GroupByError(str(e))
-            output[name] = self._try_cast(result, obj)
+            if self._transform_should_cast(how):
+                output[name] = self._try_cast(result, obj)
+            else:
+                output[name] = result
 
         if len(output) == 0:
             raise DataError('No numeric types to aggregate')
@@ -1768,6 +1772,37 @@ def cumcount(self, ascending=True):
         cumcounts = self._cumcount_array(ascending=ascending)
         return Series(cumcounts, index)
 
+    @Substitution(name='groupby')
+    @Appender(_doc_template)
+    def rank(self, method='average', ascending=True, na_option='keep',
+             pct=False, axis=0):
+        """Provides the rank of values within each group
+
+        Parameters
+        ----------
+        method : {'average', 'min', 'max', 'first', 'dense'}, efault 'average'
+            * average: average rank of group
+            * min: lowest rank in group
+            * max: highest rank in group
+            * first: ranks assigned in order they appear in the array
+            * dense: like 'min', but rank always increases by 1 between groups
+        method :  {'keep', 'top', 'bottom'}, default 'keep'
+            * keep: leave NA values where they are
+            * top: smallest rank if ascending
+            * bottom: smallest rank if descending
+        ascending : boolean, default True
+            False for ranks by high (1) to low (N)
+        pct : boolean, default False
+            Compute percentage rank of data within each group
+
+        Returns
+        -----
+        DataFrame with ranking of values within each group
+        """
+        return self._cython_transform('rank', numeric_only=False,
+                                      ties_method=method, ascending=ascending,
+                                      na_option=na_option, pct=pct, axis=axis)
+
     @Substitution(name='groupby')
     @Appender(_doc_template)
     def cumprod(self, axis=0, *args, **kwargs):
@@ -2183,6 +2218,16 @@ def get_group_levels(self):
             'cumsum': 'group_cumsum',
             'cummin': 'group_cummin',
             'cummax': 'group_cummax',
+            'rank': {
+                'name': 'group_rank',
+                'f': lambda func, a, b, c, d, **kwargs: func(
+                    a, b, c, d,
+                    kwargs.get('ties_method', 'average'),
+                    kwargs.get('ascending', True),
+                    kwargs.get('pct', False),
+                    kwargs.get('na_option', 'keep')
+                )
+            }
         }
     }
 
@@ -2242,7 +2287,8 @@ def wrapper(*args, **kwargs):
                                       (how, dtype_str))
         return func
 
-    def _cython_operation(self, kind, values, how, axis, min_count=-1):
+    def _cython_operation(self, kind, values, how, axis, min_count=-1,
+                          **kwargs):
         assert kind in ['transform', 'aggregate']
 
         # can we do this operation with our cython functions
@@ -2314,10 +2360,13 @@ def _cython_operation(self, kind, values, how, axis, min_count=-1):
             else:
                 raise
 
-        if is_numeric:
-            out_dtype = '%s%d' % (values.dtype.kind, values.dtype.itemsize)
+        if how == 'rank':
+            out_dtype = 'float'
         else:
-            out_dtype = 'object'
+            if is_numeric:
+                out_dtype = '%s%d' % (values.dtype.kind, values.dtype.itemsize)
+            else:
+                out_dtype = 'object'
 
         labels, _, _ = self.group_info
 
@@ -2334,7 +2383,8 @@ def _cython_operation(self, kind, values, how, axis, min_count=-1):
 
             # TODO: min_count
             result = self._transform(
-                result, values, labels, func, is_numeric, is_datetimelike)
+                result, values, labels, func, is_numeric, is_datetimelike,
+                **kwargs)
 
         if is_integer_dtype(result) and not is_datetimelike:
             mask = result == iNaT
@@ -2373,8 +2423,8 @@ def aggregate(self, values, how, axis=0, min_count=-1):
         return self._cython_operation('aggregate', values, how, axis,
                                       min_count=min_count)
 
-    def transform(self, values, how, axis=0):
-        return self._cython_operation('transform', values, how, axis)
+    def transform(self, values, how, axis=0, **kwargs):
+        return self._cython_operation('transform', values, how, axis, **kwargs)
 
     def _aggregate(self, result, counts, values, comp_ids, agg_func,
                    is_numeric, is_datetimelike, min_count=-1):
@@ -2394,7 +2444,7 @@ def _aggregate(self, result, counts, values, comp_ids, agg_func,
         return result
 
     def _transform(self, result, values, comp_ids, transform_func,
-                   is_numeric, is_datetimelike):
+                   is_numeric, is_datetimelike, **kwargs):
 
         comp_ids, _, ngroups = self.group_info
         if values.ndim > 3:
@@ -2406,9 +2456,9 @@ def _transform(self, result, values, comp_ids, transform_func,
 
                 chunk = chunk.squeeze()
                 transform_func(result[:, :, i], values,
-                               comp_ids, is_datetimelike)
+                               comp_ids, is_datetimelike, **kwargs)
         else:
-            transform_func(result, values, comp_ids, is_datetimelike)
+            transform_func(result, values, comp_ids, is_datetimelike, **kwargs)
 
         return result
 


Test patch:
diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py
--- a/pandas/tests/groupby/test_groupby.py
+++ b/pandas/tests/groupby/test_groupby.py
@@ -1895,6 +1895,172 @@ def test_rank_apply(self):
         expected = expected.reindex(result.index)
         assert_series_equal(result, expected)
 
+    @pytest.mark.parametrize("grps", [
+        ['qux'], ['qux', 'quux']])
+    @pytest.mark.parametrize("vals", [
+        [2, 2, 8, 2, 6],
+        [pd.Timestamp('2018-01-02'), pd.Timestamp('2018-01-02'),
+         pd.Timestamp('2018-01-08'), pd.Timestamp('2018-01-02'),
+         pd.Timestamp('2018-01-06')]])
+    @pytest.mark.parametrize("ties_method,ascending,pct,exp", [
+        ('average', True, False, [2., 2., 5., 2., 4.]),
+        ('average', True, True, [0.4, 0.4, 1.0, 0.4, 0.8]),
+        ('average', False, False, [4., 4., 1., 4., 2.]),
+        ('average', False, True, [.8, .8, .2, .8, .4]),
+        ('min', True, False, [1., 1., 5., 1., 4.]),
+        ('min', True, True, [0.2, 0.2, 1.0, 0.2, 0.8]),
+        ('min', False, False, [3., 3., 1., 3., 2.]),
+        ('min', False, True, [.6, .6, .2, .6, .4]),
+        ('max', True, False, [3., 3., 5., 3., 4.]),
+        ('max', True, True, [0.6, 0.6, 1.0, 0.6, 0.8]),
+        ('max', False, False, [5., 5., 1., 5., 2.]),
+        ('max', False, True, [1., 1., .2, 1., .4]),
+        ('first', True, False, [1., 2., 5., 3., 4.]),
+        ('first', True, True, [0.2, 0.4, 1.0, 0.6, 0.8]),
+        ('first', False, False, [3., 4., 1., 5., 2.]),
+        ('first', False, True, [.6, .8, .2, 1., .4]),
+        ('dense', True, False, [1., 1., 3., 1., 2.]),
+        ('dense', True, True, [0.2, 0.2, 0.6, 0.2, 0.4]),
+        ('dense', False, False, [3., 3., 1., 3., 2.]),
+        ('dense', False, True, [.6, .6, .2, .6, .4]),
+    ])
+    def test_rank_args(self, grps, vals, ties_method, ascending, pct, exp):
+        key = np.repeat(grps, len(vals))
+        vals = vals * len(grps)
+        df = DataFrame({'key': key, 'val': vals})
+        result = df.groupby('key').rank(method=ties_method,
+                                        ascending=ascending, pct=pct)
+
+        exp_df = DataFrame(exp * len(grps), columns=['val'])
+        assert_frame_equal(result, exp_df)
+
+    @pytest.mark.parametrize("grps", [
+        ['qux'], ['qux', 'quux']])
+    @pytest.mark.parametrize("vals", [
+        [2, 2, np.nan, 8, 2, 6, np.nan, np.nan],  # floats
+        [pd.Timestamp('2018-01-02'), pd.Timestamp('2018-01-02'), np.nan,
+         pd.Timestamp('2018-01-08'), pd.Timestamp('2018-01-02'),
+         pd.Timestamp('2018-01-06'), np.nan, np.nan]
+    ])
+    @pytest.mark.parametrize("ties_method,ascending,na_option,pct,exp", [
+        ('average', True, 'keep', False,
+            [2., 2., np.nan, 5., 2., 4., np.nan, np.nan]),
+        ('average', True, 'keep', True,
+            [0.4, 0.4, np.nan, 1.0, 0.4, 0.8, np.nan, np.nan]),
+        ('average', False, 'keep', False,
+            [4., 4., np.nan, 1., 4., 2., np.nan, np.nan]),
+        ('average', False, 'keep', True,
+            [.8, 0.8, np.nan, 0.2, 0.8, 0.4, np.nan, np.nan]),
+        ('min', True, 'keep', False,
+            [1., 1., np.nan, 5., 1., 4., np.nan, np.nan]),
+        ('min', True, 'keep', True,
+            [0.2, 0.2, np.nan, 1.0, 0.2, 0.8, np.nan, np.nan]),
+        ('min', False, 'keep', False,
+            [3., 3., np.nan, 1., 3., 2., np.nan, np.nan]),
+        ('min', False, 'keep', True,
+            [.6, 0.6, np.nan, 0.2, 0.6, 0.4, np.nan, np.nan]),
+        ('max', True, 'keep', False,
+            [3., 3., np.nan, 5., 3., 4., np.nan, np.nan]),
+        ('max', True, 'keep', True,
+            [0.6, 0.6, np.nan, 1.0, 0.6, 0.8, np.nan, np.nan]),
+        ('max', False, 'keep', False,
+            [5., 5., np.nan, 1., 5., 2., np.nan, np.nan]),
+        ('max', False, 'keep', True,
+            [1., 1., np.nan, 0.2, 1., 0.4, np.nan, np.nan]),
+        ('first', True, 'keep', False,
+            [1., 2., np.nan, 5., 3., 4., np.nan, np.nan]),
+        ('first', True, 'keep', True,
+            [0.2, 0.4, np.nan, 1.0, 0.6, 0.8, np.nan, np.nan]),
+        ('first', False, 'keep', False,
+            [3., 4., np.nan, 1., 5., 2., np.nan, np.nan]),
+        ('first', False, 'keep', True,
+            [.6, 0.8, np.nan, 0.2, 1., 0.4, np.nan, np.nan]),
+        ('dense', True, 'keep', False,
+            [1., 1., np.nan, 3., 1., 2., np.nan, np.nan]),
+        ('dense', True, 'keep', True,
+            [0.2, 0.2, np.nan, 0.6, 0.2, 0.4, np.nan, np.nan]),
+        ('dense', False, 'keep', False,
+            [3., 3., np.nan, 1., 3., 2., np.nan, np.nan]),
+        ('dense', False, 'keep', True,
+            [.6, 0.6, np.nan, 0.2, 0.6, 0.4, np.nan, np.nan]),
+        ('average', True, 'no_na', False, [2., 2., 7., 5., 2., 4., 7., 7.]),
+        ('average', True, 'no_na', True,
+            [0.25, 0.25, 0.875, 0.625, 0.25, 0.5, 0.875, 0.875]),
+        ('average', False, 'no_na', False, [4., 4., 7., 1., 4., 2., 7., 7.]),
+        ('average', False, 'no_na', True,
+            [0.5, 0.5, 0.875, 0.125, 0.5, 0.25, 0.875, 0.875]),
+        ('min', True, 'no_na', False, [1., 1., 6., 5., 1., 4., 6., 6.]),
+        ('min', True, 'no_na', True,
+            [0.125, 0.125, 0.75, 0.625, 0.125, 0.5, 0.75, 0.75]),
+        ('min', False, 'no_na', False, [3., 3., 6., 1., 3., 2., 6., 6.]),
+        ('min', False, 'no_na', True,
+            [0.375, 0.375, 0.75, 0.125, 0.375, 0.25, 0.75, 0.75]),
+        ('max', True, 'no_na', False, [3., 3., 8., 5., 3., 4., 8., 8.]),
+        ('max', True, 'no_na', True,
+            [0.375, 0.375, 1., 0.625, 0.375, 0.5, 1., 1.]),
+        ('max', False, 'no_na', False, [5., 5., 8., 1., 5., 2., 8., 8.]),
+        ('max', False, 'no_na', True,
+            [0.625, 0.625, 1., 0.125, 0.625, 0.25, 1., 1.]),
+        ('first', True, 'no_na', False, [1., 2., 6., 5., 3., 4., 7., 8.]),
+        ('first', True, 'no_na', True,
+            [0.125, 0.25, 0.75, 0.625, 0.375, 0.5, 0.875, 1.]),
+        ('first', False, 'no_na', False, [3., 4., 6., 1., 5., 2., 7., 8.]),
+        ('first', False, 'no_na', True,
+            [0.375, 0.5, 0.75, 0.125, 0.625, 0.25, 0.875, 1.]),
+        ('dense', True, 'no_na', False, [1., 1., 4., 3., 1., 2., 4., 4.]),
+        ('dense', True, 'no_na', True,
+            [0.125, 0.125, 0.5, 0.375, 0.125, 0.25, 0.5, 0.5]),
+        ('dense', False, 'no_na', False, [3., 3., 4., 1., 3., 2., 4., 4.]),
+        ('dense', False, 'no_na', True,
+            [0.375, 0.375, 0.5, 0.125, 0.375, 0.25, 0.5, 0.5])
+    ])
+    def test_rank_args_missing(self, grps, vals, ties_method, ascending,
+                               na_option, pct, exp):
+        key = np.repeat(grps, len(vals))
+        vals = vals * len(grps)
+        df = DataFrame({'key': key, 'val': vals})
+        result = df.groupby('key').rank(method=ties_method,
+                                        ascending=ascending,
+                                        na_option=na_option, pct=pct)
+
+        exp_df = DataFrame(exp * len(grps), columns=['val'])
+        assert_frame_equal(result, exp_df)
+
+    @pytest.mark.parametrize("pct,exp", [
+        (False, [3., 3., 3., 3., 3.]),
+        (True, [.6, .6, .6, .6, .6])])
+    def test_rank_resets_each_group(self, pct, exp):
+        df = DataFrame(
+            {'key': ['a', 'a', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b'],
+             'val': [1] * 10}
+        )
+        result = df.groupby('key').rank(pct=pct)
+        exp_df = DataFrame(exp * 2, columns=['val'])
+        assert_frame_equal(result, exp_df)
+
+    def test_rank_avg_even_vals(self):
+        df = DataFrame({'key': ['a'] * 4, 'val': [1] * 4})
+        result = df.groupby('key').rank()
+        exp_df = DataFrame([2.5, 2.5, 2.5, 2.5], columns=['val'])
+        assert_frame_equal(result, exp_df)
+
+    @pytest.mark.parametrize("ties_method", [
+        'average', 'min', 'max', 'first', 'dense'])
+    @pytest.mark.parametrize("ascending", [True, False])
+    @pytest.mark.parametrize("na_option", ["keep", "top", "bottom"])
+    @pytest.mark.parametrize("pct", [True, False])
+    @pytest.mark.parametrize("vals", [
+        ['bar', 'bar', 'foo', 'bar', 'baz'],
+        ['bar', np.nan, 'foo', np.nan, 'baz']
+    ])
+    def test_rank_object_raises(self, ties_method, ascending, na_option,
+                                pct, vals):
+        df = DataFrame({'key': ['foo'] * 5, 'val': vals})
+        with tm.assert_raises_regex(TypeError, "not callable"):
+            df.groupby('key').rank(method=ties_method,
+                                   ascending=ascending,
+                                   na_option=na_option, pct=pct)
+
     def test_dont_clobber_name_column(self):
         df = DataFrame({'key': ['a', 'a', 'a', 'b', 'b', 'b'],
                         'name': ['foo', 'bar', 'baz'] * 2})