Repository: dask/dask

Issue:
dask dataframe .std() produces error in cases where there are NaNs in output
I was updating the dask-examples repository to use the current version of dask in mybinder, and noticed that one of the code cells in the `dataframes.ipynb` notebook doesn't work anymore in dask 1.1.2. It could be a bug, so I wanted to flag it.

## Minimum, complete and verifiable example
dask version 1.1.2 
pandas version 0.24.1

```python
import dask
df = dask.datasets.timeseries()
df2 = df[df.y > 0]
df3 = df2.groupby('name').x.std()
```
With dask version 0.20 and pandas 0.24.1, the code doesn't throw an error.
With dask version 1.1.2 and pandas 0.24.1, I get the following error (click to expand below):
<details>
  <summary>Traceback - click to expand</summary>
  
  ```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in na_op(x, y)
   1504         try:
-> 1505             result = expressions.evaluate(op, str_rep, x, y, **eval_kwargs)
   1506         except TypeError:

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\computation\expressions.py in evaluate(op, op_str, a, b, use_numexpr, **eval_kwargs)
    207     if use_numexpr:
--> 208         return _evaluate(op, op_str, a, b, **eval_kwargs)
    209     return _evaluate_standard(op, op_str, a, b)

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\computation\expressions.py in _evaluate_standard(op, op_str, a, b, **eval_kwargs)
     67     with np.errstate(all='ignore'):
---> 68         return op(a, b)
     69 

TypeError: unsupported operand type(s) for ** or pow(): 'str' and 'int'

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in safe_na_op(lvalues, rvalues)
   1528             with np.errstate(all='ignore'):
-> 1529                 return na_op(lvalues, rvalues)
   1530         except Exception:

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in na_op(x, y)
   1506         except TypeError:
-> 1507             result = masked_arith_op(x, y, op)
   1508 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in masked_arith_op(x, y, op)
   1025             with np.errstate(all='ignore'):
-> 1026                 result[mask] = op(xrav[mask], y)
   1027 

TypeError: unsupported operand type(s) for ** or pow(): 'str' and 'int'

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\utils.py in raise_on_meta_error(funcname, udf)
    159     try:
--> 160         yield
    161     except Exception as e:

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\core.py in _emulate(func, *args, **kwargs)
   3696     with raise_on_meta_error(funcname(func), udf=kwargs.pop('udf', False)):
-> 3697         return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
   3698 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\groupby.py in _var_chunk(df, *index)
    247 
--> 248     df2 = df ** 2
    249     g2 = _groupby_raise_unaligned(df2, by=index)

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in f(self, other, axis, level, fill_value)
   2035             assert np.ndim(other) == 0
-> 2036             return self._combine_const(other, op)
   2037 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\frame.py in _combine_const(self, other, func)
   5119         assert lib.is_scalar(other) or np.ndim(other) == 0
-> 5120         return ops.dispatch_to_series(self, other, func)
   5121 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in dispatch_to_series(left, right, func, str_rep, axis)
   1156 
-> 1157     new_data = expressions.evaluate(column_op, str_rep, left, right)
   1158 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\computation\expressions.py in evaluate(op, op_str, a, b, use_numexpr, **eval_kwargs)
    207     if use_numexpr:
--> 208         return _evaluate(op, op_str, a, b, **eval_kwargs)
    209     return _evaluate_standard(op, op_str, a, b)

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\computation\expressions.py in _evaluate_standard(op, op_str, a, b, **eval_kwargs)
     67     with np.errstate(all='ignore'):
---> 68         return op(a, b)
     69 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in column_op(a, b)
   1127             return {i: func(a.iloc[:, i], b)
-> 1128                     for i in range(len(a.columns))}
   1129 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in <dictcomp>(.0)
   1127             return {i: func(a.iloc[:, i], b)
-> 1128                     for i in range(len(a.columns))}
   1129 

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in wrapper(left, right)
   1582 
-> 1583         result = safe_na_op(lvalues, rvalues)
   1584         return construct_result(left, result,

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in safe_na_op(lvalues, rvalues)
   1532                 return libalgos.arrmap_object(lvalues,
-> 1533                                               lambda x: op(x, rvalues))
   1534             raise

pandas/_libs/algos.pyx in pandas._libs.algos.arrmap()

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py in <lambda>(x)
   1532                 return libalgos.arrmap_object(lvalues,
-> 1533                                               lambda x: op(x, rvalues))
   1534             raise

TypeError: unsupported operand type(s) for ** or pow(): 'str' and 'int'

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-2-40c9ff704af9> in <module>
      2 df = dask.datasets.timeseries()
      3 df2 = df[df.y > 0]
----> 4 df3 = df2.groupby('name').x.std()
      5 df3

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\groupby.py in std(self, ddof, split_every, split_out)
    964     @derived_from(pd.core.groupby.GroupBy)
    965     def std(self, ddof=1, split_every=None, split_out=1):
--> 966         v = self.var(ddof, split_every=split_every, split_out=split_out)
    967         result = map_partitions(np.sqrt, v, meta=v)
    968         return result

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\groupby.py in var(self, ddof, split_every, split_out)
    953                      combine_kwargs={'levels': levels},
    954                      split_every=split_every, split_out=split_out,
--> 955                      split_out_setup=split_out_on_index)
    956 
    957         if isinstance(self.obj, Series):

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\core.py in apply_concat_apply(args, chunk, aggregate, combine, meta, token, chunk_kwargs, aggregate_kwargs, combine_kwargs, split_every, split_out, split_out_setup, split_out_setup_kwargs, **kwargs)
   3654 
   3655     if meta is no_default:
-> 3656         meta_chunk = _emulate(chunk, *args, udf=True, **chunk_kwargs)
   3657         meta = _emulate(aggregate, _concat([meta_chunk]), udf=True,
   3658                         **aggregate_kwargs)

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\core.py in _emulate(func, *args, **kwargs)
   3695     """
   3696     with raise_on_meta_error(funcname(func), udf=kwargs.pop('udf', False)):
-> 3697         return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
   3698 
   3699 

~\AppData\Local\conda\conda\envs\test2\lib\contextlib.py in __exit__(self, type, value, traceback)
     97                 value = type()
     98             try:
---> 99                 self.gen.throw(type, value, traceback)
    100             except StopIteration as exc:
    101                 # Suppress StopIteration *unless* it's the same exception that

~\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\utils.py in raise_on_meta_error(funcname, udf)
    175                 "{2}")
    176         msg = msg.format(" in `{0}`".format(funcname) if funcname else "", repr(e), tb)
--> 177         raise ValueError(msg)
    178 
    179 

ValueError: Metadata inference failed in `_var_chunk`.

You have supplied a custom function and Dask is unable to 
determine the type of output that that function returns. 

To resolve this please provide a meta= keyword.
The docstring of the Dask function you ran should have more information.

Original error is below:
------------------------
TypeError("unsupported operand type(s) for ** or pow(): 'str' and 'int'",)

Traceback:
---------
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\utils.py", line 160, in raise_on_meta_error
    yield
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\core.py", line 3697, in _emulate
    return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\dask\dataframe\groupby.py", line 248, in _var_chunk
    df2 = df ** 2
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py", line 2036, in f
    return self._combine_const(other, op)
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\frame.py", line 5120, in _combine_const
    return ops.dispatch_to_series(self, other, func)
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py", line 1157, in dispatch_to_series
    new_data = expressions.evaluate(column_op, str_rep, left, right)
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\computation\expressions.py", line 208, in evaluate
    return _evaluate(op, op_str, a, b, **eval_kwargs)
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\computation\expressions.py", line 68, in _evaluate_standard
    return op(a, b)
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py", line 1128, in column_op
    for i in range(len(a.columns))}
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py", line 1128, in <dictcomp>
    for i in range(len(a.columns))}
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py", line 1583, in wrapper
    result = safe_na_op(lvalues, rvalues)
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py", line 1533, in safe_na_op
    lambda x: op(x, rvalues))
  File "pandas/_libs/algos.pyx", line 690, in pandas._libs.algos.arrmap
  File "C:\Users\genevieb\AppData\Local\conda\conda\envs\test2\lib\site-packages\pandas\core\ops.py", line 1533, in <lambda>
    lambda x: op(x, rvalues))
  ```
</details>

## Further details

Similar functions to `std()`,, like `mean()` and `cumsum()` work without error. I think the problem is that there are some NaNs being produced in the calculation, and then it looks like those NaNs are interpreted as strings, which causes a problem when the next part of the calculation expects two numbers as input. 

I also suspect the issue lies somewhere in dask dataframes, because when I try the same thing with pandas (on a smaller subset of the data), I get plenty of returned NaN values in the output but it doesn't crash.
```python
import pandas as pd
df = pd.read_csv('dask_timeseries.csv')

```

pandas result:

  ```
name
Alice           NaN
Bob             NaN
Dan             NaN
Edith           NaN
Frank           NaN
Kevin           NaN
Norbert         NaN
Oliver          NaN
Ray             NaN
Sarah           NaN
Ursula     0.155848
Wendy           NaN
Name: x, dtype: float64
  ```






Discussion:

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -240,6 +240,7 @@ def _apply_chunk(df, *index, **kwargs):
 def _var_chunk(df, *index):
     if is_series_like(df):
         df = df.to_frame()
+    df = df._get_numeric_data()
     g = _groupby_raise_unaligned(df, by=index)
     x = g.sum()
 
@@ -934,8 +935,11 @@ def count(self, split_every=None, split_out=1):
 
     @derived_from(pd.core.groupby.GroupBy)
     def mean(self, split_every=None, split_out=1):
-        return (self.sum(split_every=split_every, split_out=split_out) /
-                self.count(split_every=split_every, split_out=split_out))
+        s = self.sum(split_every=split_every, split_out=split_out)
+        c = self.count(split_every=split_every, split_out=split_out)
+        if is_dataframe_like(s):
+            c = c[s.columns]
+        return s / c
 
     @derived_from(pd.core.groupby.GroupBy)
     def size(self, split_every=None, split_out=1):


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -1516,3 +1516,22 @@ def test_groupby_select_column_agg():
     actual = ddf.groupby('A')['B'].agg('var')
     expected = pdf.groupby('A')['B'].agg('var')
     assert_eq(actual, expected)
+
+
+@pytest.mark.parametrize('func', [
+    lambda x: x.std(),
+    lambda x: x.groupby('x').std(),
+    lambda x: x.groupby('x').var(),
+    lambda x: x.groupby('x').mean(),
+    lambda x: x.groupby('x').sum(),
+    lambda x: x.groupby('x').z.std(),
+])
+def test_std_object_dtype(func):
+    df = pd.DataFrame({
+        'x': [1, 2, 1],
+        'y': ['a', 'b', 'c'],
+        'z': [11., 22., 33.],
+    })
+    ddf = dd.from_pandas(df, npartitions=2)
+
+    assert_eq(func(df), func(ddf))