Repository: dask/dask

Issue:
`sum` requires dtype argument for object array.
xref: pydata/xarray#1866
`sum` requires `dtype` arguments, which is different behavior from `numpy`.
Is it intended?

```python
In [1]: import numpy as np
In [2]: import dask.array as da
In [3]: x = np.array([True, True, False, np.nan], dtype=object)
In [4]: dx = da.from_array(x, chunks=[2])
In [5]: np.sum(x)
Out[5]: nan
In [6]: dx.sum()
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-5-f94d34a739b8> in <module>()
----> 1 dx.sum()

/home/keisukefujii/Dropbox/projects/dask/dask/array/core.py in sum(self, axis, dtype, keepdims, split_every, out)
   1537         from .reductions import sum
   1538         return sum(self, axis=axis, dtype=dtype, keepdims=keepdims,
-> 1539                    split_every=split_every, out=out)
   1540 
   1541     @derived_from(np.ndarray)

/home/keisukefujii/Dropbox/projects/dask/dask/array/reductions.py in sum(a, axis, dtype, keepdims, split_every, out)
    142         dt = dtype
    143     else:
--> 144         dt = np.empty((1,), dtype=a.dtype).sum().dtype
    145     return reduction(a, chunk.sum, chunk.sum, axis=axis, keepdims=keepdims,
    146                      dtype=dt, split_every=split_every, out=out)

AttributeError: 'NoneType' object has no attribute 'dtype'
```

Python: 3.6
dask: current master



Discussion:

Resolving patch:
diff --git a/dask/array/reductions.py b/dask/array/reductions.py
--- a/dask/array/reductions.py
+++ b/dask/array/reductions.py
@@ -143,7 +143,7 @@ def sum(a, axis=None, dtype=None, keepdims=False, split_every=None, out=None):
     if dtype is not None:
         dt = dtype
     else:
-        dt = np.empty((1,), dtype=a.dtype).sum().dtype
+        dt = getattr(np.empty((1,), dtype=a.dtype).sum(), 'dtype', object)
     return reduction(a, chunk.sum, chunk.sum, axis=axis, keepdims=keepdims,
                      dtype=dt, split_every=split_every, out=out)
 
@@ -153,7 +153,7 @@ def prod(a, axis=None, dtype=None, keepdims=False, split_every=None, out=None):
     if dtype is not None:
         dt = dtype
     else:
-        dt = np.empty((1,), dtype=a.dtype).prod().dtype
+        dt = getattr(np.empty((1,), dtype=a.dtype).prod(), 'dtype', object)
     return reduction(a, chunk.prod, chunk.prod, axis=axis, keepdims=keepdims,
                      dtype=dt, split_every=split_every, out=out)
 
@@ -187,7 +187,7 @@ def nansum(a, axis=None, dtype=None, keepdims=False, split_every=None, out=None)
     if dtype is not None:
         dt = dtype
     else:
-        dt = chunk.nansum(np.empty((1,), dtype=a.dtype)).dtype
+        dt = getattr(chunk.nansum(np.empty((1,), dtype=a.dtype)), 'dtype', object)
     return reduction(a, chunk.nansum, chunk.sum, axis=axis, keepdims=keepdims,
                      dtype=dt, split_every=split_every, out=out)
 
@@ -199,7 +199,7 @@ def nanprod(a, axis=None, dtype=None, keepdims=False, split_every=None,
         if dtype is not None:
             dt = dtype
         else:
-            dt = chunk.nanprod(np.empty((1,), dtype=a.dtype)).dtype
+            dt = getattr(chunk.nansum(np.empty((1,), dtype=a.dtype)), 'dtype', object)
         return reduction(a, chunk.nanprod, chunk.prod, axis=axis,
                          keepdims=keepdims, dtype=dt, split_every=split_every,
                          out=out)
@@ -269,7 +269,7 @@ def mean(a, axis=None, dtype=None, keepdims=False, split_every=None, out=None):
     if dtype is not None:
         dt = dtype
     else:
-        dt = np.mean(np.empty(shape=(1,), dtype=a.dtype)).dtype
+        dt = getattr(np.mean(np.empty(shape=(1,), dtype=a.dtype)), 'dtype', object)
     return reduction(a, mean_chunk, mean_agg, axis=axis, keepdims=keepdims,
                      dtype=dt, split_every=split_every, combine=mean_combine,
                      out=out)
@@ -280,7 +280,7 @@ def nanmean(a, axis=None, dtype=None, keepdims=False, split_every=None,
     if dtype is not None:
         dt = dtype
     else:
-        dt = np.mean(np.empty(shape=(1,), dtype=a.dtype)).dtype
+        dt = getattr(np.mean(np.empty(shape=(1,), dtype=a.dtype)), 'dtype', object)
     return reduction(a, partial(mean_chunk, sum=chunk.nansum, numel=nannumel),
                      mean_agg, axis=axis, keepdims=keepdims, dtype=dt,
                      split_every=split_every, out=out,
@@ -377,7 +377,7 @@ def moment(a, order, axis=None, dtype=None, keepdims=False, ddof=0,
     if dtype is not None:
         dt = dtype
     else:
-        dt = np.var(np.ones(shape=(1,), dtype=a.dtype)).dtype
+        dt = getattr(np.var(np.ones(shape=(1,), dtype=a.dtype)), 'dtype', object)
     return reduction(a, partial(moment_chunk, order=order),
                      partial(moment_agg, order=order, ddof=ddof),
                      axis=axis, keepdims=keepdims,
@@ -391,7 +391,7 @@ def var(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None,
     if dtype is not None:
         dt = dtype
     else:
-        dt = np.var(np.ones(shape=(1,), dtype=a.dtype)).dtype
+        dt = getattr(np.var(np.ones(shape=(1,), dtype=a.dtype)), 'dtype', object)
     return reduction(a, moment_chunk, partial(moment_agg, ddof=ddof), axis=axis,
                      keepdims=keepdims, dtype=dt, split_every=split_every,
                      combine=moment_combine, name='var', out=out)
@@ -402,7 +402,7 @@ def nanvar(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None,
     if dtype is not None:
         dt = dtype
     else:
-        dt = np.var(np.ones(shape=(1,), dtype=a.dtype)).dtype
+        dt = getattr(np.var(np.ones(shape=(1,), dtype=a.dtype)), 'dtype', object)
     return reduction(a, partial(moment_chunk, sum=chunk.nansum, numel=nannumel),
                      partial(moment_agg, sum=np.nansum, ddof=ddof), axis=axis,
                      keepdims=keepdims, dtype=dt, split_every=split_every,
@@ -655,7 +655,7 @@ def cumreduction(func, binop, ident, x, axis=None, dtype=None, out=None):
         x = x.flatten()
         axis = 0
     if dtype is None:
-        dtype = func(np.empty((0,), dtype=x.dtype)).dtype
+        dtype = getattr(func(np.empty((0,), dtype=x.dtype)), 'dtype', object)
     assert isinstance(axis, int)
     axis = validate_axis(x.ndim, axis)
 
diff --git a/dask/array/utils.py b/dask/array/utils.py
--- a/dask/array/utils.py
+++ b/dask/array/utils.py
@@ -12,10 +12,11 @@
 from ..local import get_sync
 from ..sharedict import ShareDict
 
+
 if LooseVersion(np.__version__) >= '1.10.0':
-    allclose = np.allclose
+    _allclose = np.allclose
 else:
-    def allclose(a, b, **kwargs):
+    def _allclose(a, b, **kwargs):
         if kwargs.pop('equal_nan', False):
             a_nans = np.isnan(a)
             b_nans = np.isnan(b)
@@ -26,6 +27,16 @@ def allclose(a, b, **kwargs):
         return np.allclose(a, b, **kwargs)
 
 
+def allclose(a, b, equal_nan=False, **kwargs):
+    if getattr(a, 'dtype', None) != 'O':
+        return _allclose(a, b, equal_nan=equal_nan, **kwargs)
+    if equal_nan:
+        return (a.shape == b.shape and
+                all(np.isnan(b) if np.isnan(a) else a == b
+                    for (a, b) in zip(a.flat, b.flat)))
+    return (a == b).all()
+
+
 def same_keys(a, b):
     def key(k):
         if isinstance(k, str):
@@ -69,11 +80,15 @@ def assert_eq(a, b, check_shape=True, **kwargs):
         a = a.compute(get=get_sync)
         if hasattr(a, 'todense'):
             a = a.todense()
+        if not hasattr(a, 'dtype'):
+            a = np.array(a, dtype='O')
         if _not_empty(a):
             assert a.dtype == a_original.dtype
         if check_shape:
             assert_eq_shape(a_original.shape, a.shape, check_nan=False)
     else:
+        if not hasattr(a, 'dtype'):
+            a = np.array(a, dtype='O')
         adt = getattr(a, 'dtype', None)
 
     if isinstance(b, Array):
@@ -81,6 +96,8 @@ def assert_eq(a, b, check_shape=True, **kwargs):
         bdt = b.dtype
         _check_dsk(b.dask)
         b = b.compute(get=get_sync)
+        if not hasattr(b, 'dtype'):
+            b = np.array(b, dtype='O')
         if hasattr(b, 'todense'):
             b = b.todense()
         if _not_empty(b):
@@ -88,6 +105,8 @@ def assert_eq(a, b, check_shape=True, **kwargs):
         if check_shape:
             assert_eq_shape(b_original.shape, b.shape, check_nan=False)
     else:
+        if not hasattr(b, 'dtype'):
+            b = np.array(b, dtype='O')
         bdt = getattr(b, 'dtype', None)
 
     if str(adt) != str(bdt):


Test patch:
diff --git a/dask/array/tests/test_reductions.py b/dask/array/tests/test_reductions.py
--- a/dask/array/tests/test_reductions.py
+++ b/dask/array/tests/test_reductions.py
@@ -290,6 +290,20 @@ def test_nan():
     assert_eq(nanprod(x), da.nanprod(d))
 
 
+@pytest.mark.skipif(np.__version__ < '1.13.0', reason='nanmax/nanmin for object dtype')
+@pytest.mark.parametrize('func', ['nansum', 'sum', 'nanmin', 'min',
+                                  'nanmax', 'max'])
+def test_nan_object(func):
+    x = np.array([[1, np.nan, 3, 4],
+                  [5, 6, 7, np.nan],
+                  [9, 10, 11, 12]]).astype(object)
+    d = da.from_array(x, chunks=(2, 2))
+
+    assert_eq(getattr(np, func)(x, axis=0), getattr(da, func)(d, axis=0))
+    assert_eq(getattr(np, func)(x, axis=1), getattr(da, func)(d, axis=1))
+    assert_eq(getattr(np, func)(x), getattr(da, func)(d))
+
+
 def test_0d_array():
     x = da.mean(da.ones(4, chunks=4), axis=0).compute()
     y = np.mean(np.ones(4))