Repository: dask/dask

Issue:
Partition boundaries not honoring index
I may be misunderstanding dask, but I think the results should be independent of where the data are partitioned.  That doesn't seem to happen.  You can observe that the results change depending on whether npartitions is 2 or 1.

```
#!/usr/bin/env python
#
import dask.dataframe as dd
import pandas as pd

npartitions = 2  # change this to 1 to get correct result

outfile = '/tmp/out.csv'
df = pd.DataFrame({'uid': [1,1,2,2,2],
                   'Count': [43.,42.,43.,21.,45.]})
alldata = dd.from_pandas(df, npartitions)
alldata.set_index('uid', inplace=True)
print alldata.head()

def myfunc(gp):
    return gp['Count'].sum()

estimates = alldata.groupby('uid').apply(myfunc, columns=['uid', 'total'])
estimates.to_csv(outfile, header=True)

# print out file that just got written
infile = open(outfile, 'r')
print infile.read()
infile.close()
```



Discussion:
Some issues:
1.  `inplace=True` won't work with dask.dataframe.  We need to error loudly in this case
2.  groupby('uid') works neither with pandas nor with dask dataframe.  If you use `.index` then it works with both.

``` python
In [1]: import dask.dataframe as dd

In [2]: import pandas as pd

In [3]: 

In [3]: npartitions = 2  # change this to 1 to get correct result

In [4]: df = pd.DataFrame({'uid': [1,1,2,2,2],
   ...:                    'Count': [43.,42.,43.,21.,45.]})

In [5]: alldata = dd.from_pandas(df, npartitions)

In [6]: alldata = alldata.set_index('uid')

In [7]: alldata.compute()
Out[7]: 
     Count
uid       
1       43
1       42
2       43
2       21
2       45

In [8]: df = df.set_index('uid')

In [9]: df
Out[9]: 
     Count
uid       
1       43
1       42
2       43
2       21
2       45

In [10]: def myfunc(gp):
   ....:         return gp['Count'].sum()
   ....: 

In [11]: alldata.groupby(alldata.index).apply(myfunc, columns='Count').compute() 
Out[11]: 
uid
1     85
2    109
dtype: float64

In [12]: df.groupby(df.index).apply(myfunc)
Out[12]: 
uid
1     85
2    109
dtype: float64

```

This is still a problem though

``` python
In [13]: alldata.groupby(alldata.index).apply(myfunc, columns='Count').to_csv('/tmp/out.csv', header=True)

In [14]: !cat /tmp/out.csv
uid,0
1,85.0
2,109.0
```

Thanks!  Didn't know that about grouping by index ...

Lak


Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -1,18 +1,16 @@
 from __future__ import division
 
 import bisect
+from collections import Iterable, Iterator
+from datetime import datetime
+from distutils.version import LooseVersion
+from functools import wraps
 import operator
-import uuid
-
 from operator import getitem, setitem
 from pprint import pformat
-from functools import wraps
-from collections import Iterable
-from datetime import datetime
-from distutils.version import LooseVersion
+import uuid
 
 from toolz import merge, partial, first, partition, unique
-
 import pandas as pd
 import numpy as np
 
@@ -526,7 +524,7 @@ def f(s, n):
                 except ZeroDivisionError:
                     return np.nan
             name = '{0}mean-{1}'.format(self._token_prefix, tokenize(s))
-            return map_partitions(f, None, s, n, token=name)
+            return map_partitions(f, no_default, s, n, token=name)
 
     @wraps(pd.DataFrame.var)
     def var(self, axis=None, ddof=1):
@@ -550,7 +548,7 @@ def f(x2, x, n):
                 except ZeroDivisionError:
                     return np.nan
             name = '{0}var(ddof={1})'.format(self._token_prefix, ddof)
-            return map_partitions(f, None, x2, x, n, token=name)
+            return map_partitions(f, no_default, x2, x, n, token=name)
 
     @wraps(pd.DataFrame.std)
     def std(self, axis=None, ddof=1):
@@ -562,7 +560,7 @@ def std(self, axis=None, ddof=1):
         else:
             v = self.var(ddof=ddof)
             name = '{0}std(ddof={1})'.format(self._token_prefix, ddof)
-            return map_partitions(np.sqrt, None, v, token=name)
+            return map_partitions(np.sqrt, no_default, v, token=name)
 
     def _cum_agg(self, token, chunk, aggregate, agginit):
         """ Wrapper for cumulative operation """
@@ -1752,6 +1750,9 @@ def map_partitions(func, columns, *args, **kwargs):
     args = [from_pandas(arg, 1) if isinstance(arg, (pd.DataFrame, pd.Series))
             else arg for arg in args]
 
+    if columns is no_default:
+        columns = None
+
     from .multi import _maybe_align_partitions
     args = _maybe_align_partitions(args)
     dfs = [df for df in args if isinstance(df, _Frame)]
@@ -1761,12 +1762,27 @@ def map_partitions(func, columns, *args, **kwargs):
     for i in range(dfs[0].npartitions):
         values = [(arg._name, i if isinstance(arg, _Frame) else 0)
                   if isinstance(arg, (_Frame, Scalar)) else arg for arg in args]
-        dsk[(name, i)] = (apply, func, (tuple, values))
+        dsk[(name, i)] = (_rename, columns, (apply, func, (tuple, values)))
 
     dasks = [arg.dask for arg in args if isinstance(arg, (_Frame, Scalar))]
+
     return return_type(merge(dsk, *dasks), name, columns, args[0].divisions)
 
 
+def _rename(columns, df):
+    """ Rename columns in dataframe or series """
+    if isinstance(columns, Iterator):
+        columns = list(columns)
+    if columns is no_default:
+        return df
+    if isinstance(df, pd.DataFrame) and len(columns) == len(df.columns):
+        return df.rename(columns=dict(zip(df.columns, columns)))
+    elif isinstance(df, pd.Series):
+        return pd.Series(df, name=columns)
+    else:
+        return df
+
+
 def categorize_block(df, categories):
     """ Categorize a dataframe with given categories
 


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1868,3 +1868,16 @@ def test_to_frame():
 
     assert eq(s.to_frame(), a.to_frame())
     assert eq(s.to_frame('bar'), a.to_frame('bar'))
+
+
+def test_series_groupby_propagates_names():
+    df = pd.DataFrame({'x': [1, 2, 3], 'y': [4, 5, 6]})
+    ddf = dd.from_pandas(df, 2)
+    func = lambda df: df['y'].sum()
+
+    result = ddf.groupby('x').apply(func, columns='y')
+
+    expected = df.groupby('x').apply(func)
+    expected.name = 'y'
+
+    tm.assert_series_equal(result.compute(), expected)