Repository: dask/dask

Issue:
Random values/wrong type when grouping a filtered DataFrame by an unfiltered column
The issue has been discovered while working through `03a-DataFrame.ipynb` notebook of the tutorial.
There we have a Dask DataFrame `df`.

```python
real_df = df.compute()
# otherwise groupby issues "ValueError: cannot reindex from a duplicate axis"
real_df.index = range(len(real_df))

### using pandas
print(real_df[real_df.amount < 0].groupby(real_df.id).amount.mean())
# id
# 0     -1532.021665
# 2       -46.787321
# 4        -5.000000
# 8       -83.500000
# 11     -116.254859

### grouping by unfiltered column
# this doesn't issue any error for duplicate index items
print(df[df.amount < 0].groupby(df.id).amount.mean().compute())  
# id
# 0.0     -148.489168
# 2.0       -4.333333
# 4.0             NaN
# 8.0      -13.166667
# 11.0      -9.701362

### grouping by filtered column
print(df[df.amount < 0].groupby(df[df.amount < 0].id).amount.mean().compute())
# id
# 0     -1532.021665
# 2       -46.787321
# 4        -5.000000
# 8       -83.500000
# 11     -116.254859
```

As you can see, for some reason in Dask if you use unfiltered column for groupby, you don't receive any ValueError, you end up with `float64` column type instead of `int64` for the groupby column, and the values are very different from the ones obtained with pandas. In fact, they are different each time you run the code.


Discussion:
Thanks for the error report @p-himik !  

Can I ask you to do a bit more and provide a reproducible example?  Ideally this would be simple code that someone could copy-paste to easily get this same error.
```python
import numpy as np
import pandas as pd

import dask.dataframe as df

NAMES = ['Alice', 'Bob', 'Charlie', 'Dan', 'Edith', 'Frank', 'George',
         'Hannah', 'Ingrid', 'Jerry', 'Kevin', 'Laura', 'Michael', 'Norbert', 'Oliver',
         'Patricia', 'Quinn', 'Ray', 'Sarah', 'Tim', 'Ursula', 'Victor', 'Wendy',
         'Xavier', 'Yvonne', 'Zelda']


def account_params(k):
    ids = np.arange(k, dtype=int)
    names2 = np.random.choice(NAMES, size=k, replace=True)
    wealth_mag = np.random.exponential(100, size=k)
    wealth_trend = np.random.normal(10, 10, size=k)
    freq = np.random.exponential(size=k)
    freq /= freq.sum()

    return ids, names2, wealth_mag, wealth_trend, freq


def account_entries(n, ids, names, wealth_mag, wealth_trend, freq):
    indices = np.random.choice(ids, size=n, replace=True, p=freq)
    amounts = (np.random.normal(size=n) + wealth_trend[indices]) * wealth_mag[indices]

    return pd.DataFrame({'id': indices,
                         'names': names[indices],
                         'amount': amounts.astype('i4')},
                        columns=['id', 'names', 'amount'])


def accounts_dataframes(num_dfs, n, k):
    args = account_params(k)

    return [account_entries(n, *args) for _ in range(num_dfs)]


# interesting - n=1000, k=50 does produce the correct result for each invocation
p_dfs = accounts_dataframes(3, 1000000, 500)

d_df = df.concat([df.from_pandas(d, npartitions=1, sort=False) for d in p_dfs])

p_df = d_df.compute()
p_df.index = range(len(p_df))

print(p_df[p_df.amount < 0].groupby(p_df.id).amount.mean().head())
print(d_df[d_df.amount < 0].groupby(d_df.id).amount.mean().head())
print(d_df[d_df.amount < 0].groupby(d_df[d_df.amount < 0].id).amount.mean().head())
```
Apologies for letting this sit so long. This appears to be a bug in pandas, see https://github.com/pandas-dev/pandas/issues/15244.

The issue seems isolated to only grouping by unaligned indices. (`p_df[p_df.amount < 0]` and `p_df.id` have different indices). As such, you can avoid this bug by doing the filter beforehand (as you've noticed above).

In the long run, it'd be best to get this fixed in pandas. For now, we may be able to avoid this ourselves by aligning before the call to groupby. I'll look into this.

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -69,7 +69,7 @@ def _normalize_index(df, index):
 
     elif (isinstance(index, Series) and index.name in df.columns and
           index._name == df[index.name]._name):
-            return index.name
+        return index.name
 
     elif (isinstance(index, DataFrame) and
           set(index.columns).issubset(df.columns) and
@@ -92,7 +92,50 @@ def _maybe_slice(grouped, columns):
     return grouped
 
 
+def _is_aligned(df, by):
+    """Check if `df` and `by` have aligned indices"""
+    if isinstance(by, (pd.Series, pd.DataFrame)):
+        return df.index.equals(by.index)
+    elif isinstance(by, (list, tuple)):
+        return all(_is_aligned(df, i) for i in by)
+    else:
+        return True
+
+
+def _groupby_raise_unaligned(df, **kwargs):
+    """Groupby, but raise if df and `by` key are unaligned.
+
+    Pandas supports grouping by a column that doesn't align with the input
+    frame/series/index. However, the reindexing this causes doesn't seem to be
+    threadsafe, and can result in incorrect results. Since grouping by an
+    unaligned key is generally a bad idea, we just error loudly in dask.
+
+    For more information see pandas GH issue #15244 and Dask GH issue #1876."""
+    by = kwargs.get('by', None)
+    if by is not None and not _is_aligned(df, by):
+        msg = ("Grouping by an unaligned index is unsafe and unsupported.\n"
+               "This can be caused by filtering only one of the object or\n"
+               "grouping key. For example, the following works in pandas,\n"
+               "but not in dask:\n"
+               "\n"
+               "df[df.foo < 0].groupby(df.bar)\n"
+               "\n"
+               "This can be avoided by either filtering beforehand, or\n"
+               "passing in the name of the column instead:\n"
+               "\n"
+               "df2 = df[df.foo < 0]\n"
+               "df2.groupby(df2.bar)\n"
+               "# or\n"
+               "df[df.foo < 0].groupby('bar')\n"
+               "\n"
+               "For more information see dask GH issue #1876.")
+        raise ValueError(msg)
+    return df.groupby(**kwargs)
+
+
 def _groupby_slice_apply(df, grouper, key, func):
+    # No need to use raise if unaligned here - this is only called after
+    # shuffling, which makes everything aligned already
     g = df.groupby(grouper)
     if key:
         g = g[key]
@@ -101,7 +144,7 @@ def _groupby_slice_apply(df, grouper, key, func):
 
 def _groupby_get_group(df, by_key, get_key, columns):
     # SeriesGroupBy may pass df which includes group key
-    grouped = df.groupby(by_key)
+    grouped = _groupby_raise_unaligned(df, by=by_key)
 
     if get_key in grouped.groups:
         if isinstance(df, pd.DataFrame):
@@ -129,18 +172,20 @@ def _apply_chunk(df, *index, **kwargs):
     func = kwargs.pop('chunk')
     columns = kwargs.pop('columns')
 
+    g = _groupby_raise_unaligned(df, by=index)
+
     if isinstance(df, pd.Series) or columns is None:
-        return func(df.groupby(index))
+        return func(g)
     else:
         if isinstance(columns, (tuple, list, set, pd.Index)):
             columns = list(columns)
-        return func(df.groupby(index)[columns])
+        return func(g[columns])
 
 
 def _var_chunk(df, *index):
     if isinstance(df, pd.Series):
         df = df.to_frame()
-    g = df.groupby(index)
+    g = _groupby_raise_unaligned(df, by=index)
     x = g.sum()
     x2 = g.agg(lambda x: (x**2).sum()).rename(columns=lambda c: c + '-x2')
     n = g.count().rename(columns=lambda c: c + '-count')
@@ -176,10 +221,11 @@ def _nunique_df_chunk(df, *index, **kwargs):
     levels = kwargs.pop('levels')
     name = kwargs.pop('name')
 
-    # we call set_index here to force a possibly duplicate index
-    # for our reduce step
-    grouped = df.groupby(index)[[name]].apply(pd.DataFrame.drop_duplicates)
+    g = _groupby_raise_unaligned(df, by=index)
+    grouped = g[[name]].apply(pd.DataFrame.drop_duplicates)
 
+    # we set the index here to force a possibly duplicate index
+    # for our reduce step
     if isinstance(levels, list):
         grouped.index = pd.MultiIndex.from_arrays([
             grouped.index.get_level_values(level=level) for level in levels
@@ -478,7 +524,7 @@ def _groupby_apply_funcs(df, *index, **kwargs):
         kwargs.update(by=list(index))
 
     funcs = kwargs.pop('funcs')
-    grouped = df.groupby(**kwargs)
+    grouped = _groupby_raise_unaligned(df, **kwargs)
 
     result = collections.OrderedDict()
     for result_column, func, func_kwargs in funcs:


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -998,3 +998,38 @@ def test_cumulative_axis1(func):
     ddf = dd.from_pandas(df, npartitions=4)
     assert_eq(getattr(df.groupby('a'), func)(axis=1),
               getattr(ddf.groupby('a'), func)(axis=1))
+
+
+def test_groupby_unaligned_index():
+    df = pd.DataFrame({'a': np.random.randint(0, 10, 50),
+                       'b': np.random.randn(50),
+                       'c': np.random.randn(50)})
+    ddf = dd.from_pandas(df, npartitions=5)
+    filtered = df[df.b < 0.5]
+    dfiltered = ddf[ddf.b < 0.5]
+
+    ddf_group = dfiltered.groupby(ddf.a)
+    ds_group = dfiltered.b.groupby(ddf.a)
+
+    bad = [ddf_group.mean(),
+           ddf_group.var(),
+           ddf_group.b.nunique(),
+           ddf_group.get_group(0),
+           ds_group.mean(),
+           ds_group.var(),
+           ds_group.nunique(),
+           ds_group.get_group(0)]
+
+    for obj in bad:
+        with pytest.raises(ValueError):
+            obj.compute()
+
+    def add1(x):
+        return x + 1
+
+    df_group = filtered.groupby(df.a)
+    good = [(ddf_group.apply(add1, meta=ddf), df_group.apply(add1)),
+            (ddf_group.b.apply(add1, meta=ddf.b), df_group.b.apply(add1))]
+
+    for (res, sol) in good:
+        assert_eq(res, sol)