Repository: dask/dask

Issue:
Test failures with upstream dev
We have some test failures with upstream dev right now:
https://travis-ci.org/dask/dask/jobs/443796458

It would be good to figure out what's going on. I suspect these are related in some way to the recent addition of `__array_function__` overloads to NumPy functions (https://github.com/numpy/numpy/issues/12028).


Discussion:
This was at least partially due to https://github.com/numpy/numpy/issues/12225
I think it's entirely due to https://github.com/numpy/numpy/issues/12225
The offending lines seem to be this check in `dask.array.chunk`:
https://github.com/dask/dask/blob/0910b41a536e11a9216f687c7ac8ccb75b2909a9/dask/array/chunk.py#L32-L33

Git blame shows this was added way back in https://github.com/dask/dask/pull/73 to ensure compatibility with NumPy 1.7. I don't know what version of NumPy we require now for dask.array but 1.7 is really old (4+ years). At this point, I think we can just drop `keepdims_wrapper` entirely.
It appears that argmax/min and nanargmax/min do not support the keepdims
keyword

On Tue, Oct 23, 2018 at 6:05 PM Stephan Hoyer <notifications@github.com>
wrote:

> The offending lines seem to be this check in dask.array.chunk:
>
> https://github.com/dask/dask/blob/0910b41a536e11a9216f687c7ac8ccb75b2909a9/dask/array/chunk.py#L32-L33
>
> Git blame shows this was added way back in #73
> <https://github.com/dask/dask/pull/73> to ensure compatibility with NumPy
> 1.7. I don't know what version of NumPy we require now for dask.array but
> 1.7 is really old (4+ years). At this point, I think we can just drop
> keepdims_wrapper entirely.
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/4111#issuecomment-432438840>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszND7SrQDUohK4MiP6Ybu64QpzYk1ks5un5KMgaJpZM4Xxovl>
> .
>

We could special case those though

On Tue, Oct 23, 2018 at 6:15 PM Matthew Rocklin <mrocklin@anaconda.com>
wrote:

> It appears that argmax/min and nanargmax/min do not support the keepdims
> keyword
>
> On Tue, Oct 23, 2018 at 6:05 PM Stephan Hoyer <notifications@github.com>
> wrote:
>
>> The offending lines seem to be this check in dask.array.chunk:
>>
>> https://github.com/dask/dask/blob/0910b41a536e11a9216f687c7ac8ccb75b2909a9/dask/array/chunk.py#L32-L33
>>
>> Git blame shows this was added way back in #73
>> <https://github.com/dask/dask/pull/73> to ensure compatibility with
>> NumPy 1.7. I don't know what version of NumPy we require now for dask.array
>> but 1.7 is really old (4+ years). At this point, I think we can just drop
>> keepdims_wrapper entirely.
>>
>> —
>> You are receiving this because you commented.
>> Reply to this email directly, view it on GitHub
>> <https://github.com/dask/dask/issues/4111#issuecomment-432438840>, or mute
>> the thread
>> <https://github.com/notifications/unsubscribe-auth/AASszND7SrQDUohK4MiP6Ybu64QpzYk1ks5un5KMgaJpZM4Xxovl>
>> .
>>
>

Another option would be check for `func.__wrapped__` inside dask's wrapper for `getargspec()`. But I do think it's a little cleaner not to rely on introspection here.

Resolving patch:
diff --git a/dask/array/chunk.py b/dask/array/chunk.py
--- a/dask/array/chunk.py
+++ b/dask/array/chunk.py
@@ -7,7 +7,7 @@
 import numpy as np
 from . import numpy_compat as npcompat
 
-from ..compatibility import getargspec, Container, Iterable, Sequence
+from ..compatibility import Container, Iterable, Sequence
 from ..core import flatten
 from ..utils import ignoring
 
@@ -29,9 +29,6 @@ def keepdims_wrapper(a_callable):
     A wrapper for functions that don't provide keepdims to ensure that they do.
     """
 
-    if "keepdims" in getargspec(a_callable).args:
-        return a_callable
-
     @wraps(a_callable)
     def keepdims_wrapped_callable(x, axis=None, keepdims=None, *args, **kwargs):
         r = a_callable(x, axis=axis, *args, **kwargs)
@@ -62,18 +59,18 @@ def keepdims_wrapped_callable(x, axis=None, keepdims=None, *args, **kwargs):
 
 
 # Wrap NumPy functions to ensure they provide keepdims.
-sum = keepdims_wrapper(np.sum)
-prod = keepdims_wrapper(np.prod)
-min = keepdims_wrapper(np.min)
-max = keepdims_wrapper(np.max)
+sum = np.sum
+prod = np.prod
+min = np.min
+max = np.max
 argmin = keepdims_wrapper(np.argmin)
 nanargmin = keepdims_wrapper(np.nanargmin)
 argmax = keepdims_wrapper(np.argmax)
 nanargmax = keepdims_wrapper(np.nanargmax)
-any = keepdims_wrapper(np.any)
-all = keepdims_wrapper(np.all)
-nansum = keepdims_wrapper(np.nansum)
-nanprod = keepdims_wrapper(np.nanprod)
+any = np.any
+all = np.all
+nansum = np.nansum
+nanprod = np.nanprod
 
 try:
     from numpy import nancumprod, nancumsum
@@ -81,25 +78,22 @@ def keepdims_wrapped_callable(x, axis=None, keepdims=None, *args, **kwargs):
     nancumprod = npcompat.nancumprod
     nancumsum = npcompat.nancumsum
 
-nancumprod = keepdims_wrapper(nancumprod)
-nancumsum = keepdims_wrapper(nancumsum)
-
-nanmin = keepdims_wrapper(np.nanmin)
-nanmax = keepdims_wrapper(np.nanmax)
-mean = keepdims_wrapper(np.mean)
+nanmin = np.nanmin
+nanmax = np.nanmax
+mean = np.mean
 
 with ignoring(AttributeError):
-    nanmean = keepdims_wrapper(np.nanmean)
+    nanmean = np.nanmean
 
-var = keepdims_wrapper(np.var)
+var = np.var
 
 with ignoring(AttributeError):
-    nanvar = keepdims_wrapper(np.nanvar)
+    nanvar = np.nanvar
 
-std = keepdims_wrapper(np.std)
+std = np.std
 
 with ignoring(AttributeError):
-    nanstd = keepdims_wrapper(np.nanstd)
+    nanstd = np.nanstd
 
 
 def coarsen(reduction, x, axes, trim_excess=False):
diff --git a/dask/compatibility.py b/dask/compatibility.py
--- a/dask/compatibility.py
+++ b/dask/compatibility.py
@@ -304,14 +304,15 @@ def writelines(self, *args, **kwargs):
 
 
 def getargspec(func):
-    """Version of inspect.getargspec that works for functools.partial objects"""
+    """Version of inspect.getargspec that works with partial and warps."""
     if isinstance(func, functools.partial):
-        return _getargspec(func.func)
+        return getargspec(func.func)
+ 
+    func = getattr(func, '__wrapped__', func)
+    if isinstance(func, type):
+        return _getargspec(func.__init__)
     else:
-        if isinstance(func, type):
-            return _getargspec(func.__init__)
-        else:
-            return _getargspec(func)
+        return _getargspec(func)
 
 
 def bind_method(cls, name, func):


Test patch:
diff --git a/dask/tests/test_compatibility.py b/dask/tests/test_compatibility.py
new file mode 100644
--- /dev/null
+++ b/dask/tests/test_compatibility.py
@@ -0,0 +1,24 @@
+import functools
+
+from dask.compatibility import getargspec
+
+
+def test_getargspec():
+
+    def func(x, y):
+        pass
+    assert getargspec(func).args == ['x', 'y']
+
+    func2 = functools.partial(func, 2)
+    # this is a bit of a lie, but maybe close enough
+    assert getargspec(func2).args == ['x', 'y']
+
+    def wrapper(*args, **kwargs):
+        pass
+    wrapper.__wrapped__ = func
+    assert getargspec(wrapper).args == ['x', 'y']
+
+    class MyType(object):
+        def __init__(self, x, y):
+            pass
+    assert getargspec(MyType).args == ['self', 'x', 'y']