Repository: dask/dask

Issue:
sum() got an unexpected keyword argument 'keepdims'
I'm not sure if this is related to #4311 because in my case it only occurs using Dask.

Also, before, I was on Dask 1.2.0 and this error was inexistent:

- Dask 2.0.0+11.g7f26af69
- Pandas 0.24.2
- NumPy 1.16.4

```python
import pandas as pd
import dask.dataframe as dd
from dask.array import stats as dstats
from scipy import stats as sstats

s = pd.Series(list(range(1,10000)), name='series', dtype=float)
sd = dd.from_pandas(s, npartitions=2)

sstats.moment(s, 2, 0)
# 8331666.666666667

sstats.moment(sd, 2, 0)
# 8331666.666666667

dstats.moment(sd, 2, 0)
```

```
TypeError                                 Traceback (most recent call last)
<ipython-input-5-200bbf2f1719> in <module>
      1 
      2 
----> 3 dstats.moment(sd, 2, 0)

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/dask/array/stats.py in moment(a, moment, axis, nan_policy)
    364             "`nan_policy` other than 'propagate' " "have not been implemented."
    365         )
--> 366     return da.moment(a, moment, axis=axis)
    367 
    368 

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/dask/array/reductions.py in moment(a, order, axis, dtype, keepdims, ddof, split_every, out)
    740         out=out,
    741         concatenate=False,
--> 742         combine=partial(moment_combine, order=order),
    743     )
    744 

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/dask/array/reductions.py in reduction(x, chunk, aggregate, axis, keepdims, dtype, split_every, combine, name, out, concatenate, output_size, meta)
    156     # The dtype of `tmp` doesn't actually matter, and may be incorrect.
    157     tmp = blockwise(
--> 158         chunk, inds, x, inds, axis=axis, keepdims=True, dtype=dtype or float
    159     )
    160     tmp._chunks = tuple(

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/dask/array/blockwise.py in blockwise(func, out_ind, name, token, dtype, adjust_chunks, new_axes, align_arrays, concatenate, meta, *args, **kwargs)
    231         from .utils import compute_meta
    232 
--> 233         meta = compute_meta(func, dtype, *args[::2], **kwargs)
    234     if meta is not None:
    235         return Array(graph, out, chunks, meta=meta)

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/dask/array/utils.py in compute_meta(func, _dtype, *args, **kwargs)
    120     else:
    121         try:
--> 122             meta = func(*args_meta, **kwargs_meta)
    123         except TypeError as e:
    124             if (

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/dask/array/reductions.py in moment_chunk(A, order, sum, numel, dtype, meta, **kwargs)
    624 
    625     n = n.astype(np.int64)
--> 626     total = sum(A, dtype=dtype, **kwargs)
    627     with np.errstate(divide="ignore", invalid="ignore"):
    628         u = total / n

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/numpy/core/fromnumeric.py in sum(a, axis, dtype, out, keepdims, initial)
   2074 
   2075     return _wrapreduction(a, np.add, 'sum', axis, dtype, out, keepdims=keepdims,
-> 2076                           initial=initial)
   2077 
   2078 

~/anaconda/envs/tarsierenv/lib/python3.7/site-packages/numpy/core/fromnumeric.py in _wrapreduction(obj, ufunc, method, axis, dtype, out, **kwargs)
     80             # support a dtype.
     81             if dtype is not None:
---> 82                 return reduction(axis=axis, dtype=dtype, out=out, **passkwargs)
     83             else:
     84                 return reduction(axis=axis, out=out, **passkwargs)

TypeError: sum() got an unexpected keyword argument 'keepdims'
```

Any ideas what the problem may be?
da.mean crashing with numpy ndarray as input
Hi,

The following code crashes when run interactively in a fresh virtualenv:
```python
import numpy as np
import dask.array as da
arr = np.array([0.28588], dtype=np.float32)
da.mean(arr)
```

The traceback says:
```python-traceback
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/a001673/daskbug/lib/python3.6/site-packages/dask/array/core.py", line 1114, in __repr__
    self.dtype,
  File "/home/a001673/daskbug/lib/python3.6/site-packages/dask/array/core.py", line 1044, in dtype
    return self._meta.dtype
AttributeError: 'dict' object has no attribute 'dtype'
```

Versions tested on:
 * numpy 1.16.4
 * dask 2.0.0
 * toolz 0.9.0
 * python 3.6.4 and 3.7.3

Works when installing dask 1.2.2 instead






Discussion:
It looks like this is failing during meta computation because the `pandas.Series.sum` method doesn't accept the `keepdims` argument, but the dask array method uses it.

If we were to ignore the `meta=` failure then things here would work out, but it's not clear yet where to do that cleanly.  I can take another look tomorrow.
Just FYI, on pandas master this works

```python
dstats.moment(sd, 2, 0)
## -- End pasted text --
Out[1]: dask.array<moment_agg-aggregate, shape=(), dtype=float64, chunksize=()>

In [2]: dstats.moment(sd, 2, 0).compute()
Out[2]: 8331666.666666667
```

pandas DataFrame.sum now accepts `keepdims`.

A pandas release is imminent. We're supposed to do a release candidate today (though it may not happen).
I know how to fix this, will send a PR soon.


Resolving patch:
diff --git a/dask/array/reductions.py b/dask/array/reductions.py
--- a/dask/array/reductions.py
+++ b/dask/array/reductions.py
@@ -164,7 +164,7 @@ def reduction(
     if meta is None and hasattr(x, "_meta"):
         try:
             reduced_meta = compute_meta(
-                chunk, x.dtype, x._meta, axis=axis, keepdims=True, meta=True
+                chunk, x.dtype, x._meta, axis=axis, keepdims=True, computing_meta=True
             )
         except TypeError:
             reduced_meta = compute_meta(
@@ -301,7 +301,7 @@ def partial_reduce(
     meta = x._meta
     if reduced_meta is not None:
         try:
-            meta = func(reduced_meta, meta=True)
+            meta = func(reduced_meta, computing_meta=True)
         # no meta keyword argument exists for func, and it isn't required
         except TypeError:
             meta = func(reduced_meta)
@@ -532,8 +532,10 @@ def nannumel(x, **kwargs):
     return chunk.sum(~np.isnan(x), **kwargs)
 
 
-def mean_chunk(x, sum=chunk.sum, numel=numel, dtype="f8", meta=False, **kwargs):
-    if meta:
+def mean_chunk(
+    x, sum=chunk.sum, numel=numel, dtype="f8", computing_meta=False, **kwargs
+):
+    if computing_meta:
         return x
     n = numel(x, dtype=dtype, **kwargs)
 
@@ -543,15 +545,21 @@ def mean_chunk(x, sum=chunk.sum, numel=numel, dtype="f8", meta=False, **kwargs):
 
 
 def mean_combine(
-    pairs, sum=chunk.sum, numel=numel, dtype="f8", axis=None, meta=False, **kwargs
+    pairs,
+    sum=chunk.sum,
+    numel=numel,
+    dtype="f8",
+    axis=None,
+    computing_meta=False,
+    **kwargs
 ):
     if not isinstance(pairs, list):
         pairs = [pairs]
 
-    ns = deepmap(lambda pair: pair["n"], pairs) if not meta else pairs
+    ns = deepmap(lambda pair: pair["n"], pairs) if not computing_meta else pairs
     n = _concatenate2(ns, axes=axis).sum(axis=axis, **kwargs)
 
-    if meta:
+    if computing_meta:
         return n
 
     totals = deepmap(lambda pair: pair["total"], pairs)
@@ -560,12 +568,12 @@ def mean_combine(
     return {"n": n, "total": total}
 
 
-def mean_agg(pairs, dtype="f8", axis=None, meta=False, **kwargs):
-    ns = deepmap(lambda pair: pair["n"], pairs) if not meta else pairs
+def mean_agg(pairs, dtype="f8", axis=None, computing_meta=False, **kwargs):
+    ns = deepmap(lambda pair: pair["n"], pairs) if not computing_meta else pairs
     n = _concatenate2(ns, axes=axis)
     n = np.sum(n, axis=axis, dtype=dtype, **kwargs)
 
-    if meta:
+    if computing_meta:
         return n
 
     totals = deepmap(lambda pair: pair["total"], pairs)
@@ -618,9 +626,9 @@ def nanmean(a, axis=None, dtype=None, keepdims=False, split_every=None, out=None
 
 
 def moment_chunk(
-    A, order=2, sum=chunk.sum, numel=numel, dtype="f8", meta=False, **kwargs
+    A, order=2, sum=chunk.sum, numel=numel, dtype="f8", computing_meta=False, **kwargs
 ):
-    if meta:
+    if computing_meta:
         return A
     n = numel(A, **kwargs)
 
@@ -644,7 +652,14 @@ def _moment_helper(Ms, ns, inner_term, order, sum, axis, kwargs):
 
 
 def moment_combine(
-    pairs, order=2, ddof=0, dtype="f8", sum=np.sum, axis=None, meta=False, **kwargs
+    pairs,
+    order=2,
+    ddof=0,
+    dtype="f8",
+    sum=np.sum,
+    axis=None,
+    computing_meta=False,
+    **kwargs
 ):
     if not isinstance(pairs, list):
         pairs = [pairs]
@@ -652,11 +667,11 @@ def moment_combine(
     kwargs["dtype"] = dtype
     kwargs["keepdims"] = True
 
-    ns = deepmap(lambda pair: pair["n"], pairs) if not meta else pairs
+    ns = deepmap(lambda pair: pair["n"], pairs) if not computing_meta else pairs
     ns = _concatenate2(ns, axes=axis)
     n = ns.sum(axis=axis, **kwargs)
 
-    if meta:
+    if computing_meta:
         return n
 
     totals = _concatenate2(deepmap(lambda pair: pair["total"], pairs), axes=axis)
@@ -675,7 +690,14 @@ def moment_combine(
 
 
 def moment_agg(
-    pairs, order=2, ddof=0, dtype="f8", sum=np.sum, axis=None, meta=False, **kwargs
+    pairs,
+    order=2,
+    ddof=0,
+    dtype="f8",
+    sum=np.sum,
+    axis=None,
+    computing_meta=False,
+    **kwargs
 ):
     if not isinstance(pairs, list):
         pairs = [pairs]
@@ -686,11 +708,11 @@ def moment_agg(
     keepdim_kw = kwargs.copy()
     keepdim_kw["keepdims"] = True
 
-    ns = deepmap(lambda pair: pair["n"], pairs) if not meta else pairs
+    ns = deepmap(lambda pair: pair["n"], pairs) if not computing_meta else pairs
     ns = _concatenate2(ns, axes=axis)
     n = ns.sum(axis=axis, **keepdim_kw)
 
-    if meta:
+    if computing_meta:
         return n
 
     totals = _concatenate2(deepmap(lambda pair: pair["total"], pairs), axes=axis)
diff --git a/dask/array/utils.py b/dask/array/utils.py
--- a/dask/array/utils.py
+++ b/dask/array/utils.py
@@ -11,7 +11,7 @@
 
 from .core import Array
 from ..highlevelgraph import HighLevelGraph
-from ..utils import ignoring, is_arraylike
+from ..utils import has_keyword, ignoring, is_arraylike
 
 try:
     AxisError = np.AxisError
@@ -119,6 +119,9 @@ def compute_meta(func, _dtype, *args, **kwargs):
         meta = func(*args_meta)
     else:
         try:
+            # some reduction functions need to know they are computing meta
+            if has_keyword(func, "computing_meta"):
+                kwargs_meta["computing_meta"] = True
             meta = func(*args_meta, **kwargs_meta)
         except TypeError as e:
             if (


Test patch:
diff --git a/dask/dataframe/tests/test_arithmetics_reduction.py b/dask/dataframe/tests/test_arithmetics_reduction.py
--- a/dask/dataframe/tests/test_arithmetics_reduction.py
+++ b/dask/dataframe/tests/test_arithmetics_reduction.py
@@ -1335,3 +1335,14 @@ def test_divmod():
     expected = divmod(df1, df2)
     assert_eq(result[0], expected[0])
     assert_eq(result[1], expected[1])
+
+
+def test_moment():
+    scipy = pytest.importorskip("scipy")
+    from dask.array import stats
+    from dask.array.utils import assert_eq
+
+    df = pd.Series(list(range(10)))
+    ddf = dd.from_pandas(df, npartitions=2)
+
+    assert_eq(stats.moment(ddf, 2, 0), scipy.stats.moment(df, 2, 0))