Repository: pandas-dev/pandas

Issue:
BUG: Binned groupby median function calculates median on empty bins and outputs random numbers
#### Code Sample, a copy-pastable example if possible

```
import pandas as pd
d = pd.DataFrame([1,2,5,6,9,3,6,5,9,7,11,36,4,7,8,25,8,24])
b = [0,5,10,15,20,25,30,35,40,45,50,55]
g = d.groupby(pd.cut(d[0],b))
print g.mean()
print g.median()
print g.get_group('(0, 5]').median()
print g.get_group('(40, 45]').median()
```
#### Expected Output

```
                  0
0                  
(0, 5]     3.333333
(5, 10]    7.500000
(10, 15]  11.000000
(15, 20]        NaN
(20, 25]  24.500000
(25, 30]        NaN
(30, 35]        NaN
(35, 40]  36.000000
(40, 45]        NaN
(45, 50]        NaN
(50, 55]        NaN
             0
0             
(0, 5]     3.5
(5, 10]    7.5
(10, 15]  11.0
(15, 20]  18.0
(20, 25]  24.5
(25, 30]  30.5
(30, 35]  30.5
(35, 40]  36.0
(40, 45]  18.0
(45, 50]  18.0
(50, 55]  18.0
0    3.5
dtype: float64
Traceback (most recent call last):

  File "<ipython-input-9-0663486889da>", line 1, in <module>
    runfile('C:/PythonDir/test04.py', wdir='C:/PythonDir')

  File "C:\Anaconda2\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 714, in runfile
    execfile(filename, namespace)

  File "C:\Anaconda2\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 74, in execfile
    exec(compile(scripttext, filename, 'exec'), glob, loc)

  File "C:/PythonDir/test04.py", line 20, in <module>
    print g.get_group('(40, 45]').median()

  File "C:\Anaconda2\lib\site-packages\pandas\core\groupby.py", line 587, in get_group
    raise KeyError(name)

KeyError: '(40, 45]'
```

This example shows how the median-function of the groupby object outputs a random number instead of NaN like the mean-function does when a bin is empty. Directly trying to call that bin with its key leads to an error since it doesn't exist, yet the full median output suggests it does exist and that the value might even be meaningful (like in the (15, 20] bin or the (30, 35] bin). The wrong numbers that are returned can change randomly, another possible output using the same code might look like this:

```
(0, 5]     3.500000e+00
(5, 10]    7.500000e+00
(10, 15]   1.100000e+01
(15, 20]   1.800000e+01
(20, 25]   2.450000e+01
(25, 30]   3.050000e+01
(30, 35]   3.050000e+01
(35, 40]   3.600000e+01
(40, 45]  4.927210e+165
(45, 50]  4.927210e+165
(50, 55]  4.927210e+165
```
#### output of `pd.show_versions()`

`pandas: 0.18.1`



Discussion:
@Khris777 Thanks for reporting!

As a workaround for now, you can do:

```
In [11]: g.agg(lambda x: x.median())
Out[11]:
             0
0
(0, 5]     3.5
(5, 10]    7.5
(10, 15]  11.0
(15, 20]   NaN
(20, 25]  24.5
(25, 30]   NaN
(30, 35]   NaN
(35, 40]  36.0
(40, 45]   NaN
(45, 50]   NaN
(50, 55]   NaN
```

First time contributor, thought I'd take a look into this one. Do you think there's a more logical response than raising a KeyError to `g.get_group('(40, 45]')`  ?

get_group with no additional arguments is supposed to return a subset of the original dataframe with values that fall within the specified interval. If there are no values in the interval (40,45] in the original dataframe, there's no way to slice that up into a sensible response. Empty dataframe?

ATM, internval types are actual string reprs (and not a distinct dtype), so yes, `g.get_group('(40, 45)')` should be a `KeyError`, just like any other indexing operation.


Resolving patch:
diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -4424,12 +4424,13 @@ def _reorder_by_uniques(uniques, labels):
 def _groupby_indices(values):
 
     if is_categorical_dtype(values):
-
         # we have a categorical, so we can do quite a bit
         # bit better than factorizing again
         reverse = dict(enumerate(values.categories))
         codes = values.codes.astype('int64')
-        _, counts = _hash.value_count_int64(codes, False)
+
+        mask = 0 <= codes
+        counts = np.bincount(codes[mask], minlength=values.categories.size)
     else:
         reverse, codes, counts = _algos.group_labels(
             _values_from_object(_ensure_object(values)))


Test patch:
diff --git a/pandas/tests/test_groupby.py b/pandas/tests/test_groupby.py
--- a/pandas/tests/test_groupby.py
+++ b/pandas/tests/test_groupby.py
@@ -799,6 +799,17 @@ def test_get_group(self):
         self.assertRaises(ValueError,
                           lambda: g.get_group(('foo', 'bar', 'baz')))
 
+    def test_get_group_empty_bins(self):
+        d = pd.DataFrame([3, 1, 7, 6])
+        bins = [0, 5, 10, 15]
+        g = d.groupby(pd.cut(d[0], bins))
+
+        result = g.get_group('(0, 5]')
+        expected = DataFrame([3, 1], index=[0, 1])
+        assert_frame_equal(result, expected)
+
+        self.assertRaises(KeyError, lambda: g.get_group('(10, 15]'))
+
     def test_get_group_grouped_by_tuple(self):
         # GH 8121
         df = DataFrame([[(1, ), (1, 2), (1, ), (1, 2)]], index=['ids']).T
@@ -4415,6 +4426,16 @@ def test_cython_median(self):
         xp = df.groupby(labels).median()
         assert_frame_equal(rs, xp)
 
+    def test_median_empty_bins(self):
+        df = pd.DataFrame(np.random.randint(0, 44, 500))
+
+        grps = range(0, 55, 5)
+        bins = pd.cut(df[0], grps)
+
+        result = df.groupby(bins).median()
+        expected = df.groupby(bins).agg(lambda x: x.median())
+        assert_frame_equal(result, expected)
+
     def test_groupby_categorical_no_compress(self):
         data = Series(np.random.randn(9))
 
@@ -6123,6 +6144,27 @@ def test__cython_agg_general(self):
                 exc.args += ('operation: %s' % op, )
                 raise
 
+    def test_cython_agg_empty_buckets(self):
+        ops = [('mean', np.mean),
+               ('median', np.median),
+               ('var', lambda x: np.var(x, ddof=1)),
+               ('add', lambda x: np.sum(x) if len(x) > 0 else np.nan),
+               ('prod', np.prod),
+               ('min', np.min),
+               ('max', np.max), ]
+
+        df = pd.DataFrame([11, 12, 13])
+        grps = range(0, 55, 5)
+
+        for op, targop in ops:
+            result = df.groupby(pd.cut(df[0], grps))._cython_agg_general(op)
+            expected = df.groupby(pd.cut(df[0], grps)).agg(lambda x: targop(x))
+            try:
+                tm.assert_frame_equal(result, expected)
+            except BaseException as exc:
+                exc.args += ('operation: %s' % op,)
+                raise
+
     def test_cython_group_transform_algos(self):
         # GH 4095
         dtypes = [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint32,