Repository: dask/dask

Issue:
ValueError: `dtype` inference failed in `map_blocks`.
After upgrading to dask 0.13 from 0.11 I am getting errors relating to failed dtype inference within `map_blocks`. Here's an example:

```
Python 3.6.0 | packaged by conda-forge | (default, Jan 13 2017, 23:17:12) 
[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import dask.array as da
>>> import dask
>>> dask.__version__
'0.13.0'
>>> import numpy as np
>>> np.__version__
'1.11.3'
>>> x = np.ones((5, 3))
>>> y = np.ones((5, 3))
>>> dx = da.from_array(x, chunks=(2, None))
>>> dy = da.from_array(y, chunks=(2, None))
>>> def f(a, b):
...     return a + b[:, 1, None]
... 
>>> dz = da.map_blocks(f, dx, dy, dtype=x.dtype)
>>> dz.compute()
array([[ 2.,  2.,  2.],
       [ 2.,  2.,  2.],
       [ 2.,  2.,  2.],
       [ 2.,  2.,  2.],
       [ 2.,  2.,  2.]])
>>> dz = da.map_blocks(f, dx, dy)
Traceback (most recent call last):
  File "/home/aliman/miniconda3/envs/dev-scikit-allel-py36/lib/python3.6/site-packages/dask/array/core.py", line 457, in apply_infer_dtype
    o = func(*args, **kwargs)
  File "<stdin>", line 2, in f
IndexError: index 1 is out of bounds for axis 1 with size 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/aliman/miniconda3/envs/dev-scikit-allel-py36/lib/python3.6/site-packages/dask/array/core.py", line 629, in map_blocks
    dtype = apply_infer_dtype(func, args, kwargs2, 'map_blocks')
  File "/home/aliman/miniconda3/envs/dev-scikit-allel-py36/lib/python3.6/site-packages/dask/array/core.py", line 470, in apply_infer_dtype
    raise ValueError(msg)
ValueError: `dtype` inference failed in `map_blocks`.

Original error is below:
------------------------
IndexError('index 1 is out of bounds for axis 1 with size 1',)

Traceback:
---------
  File "/home/aliman/miniconda3/envs/dev-scikit-allel-py36/lib/python3.6/site-packages/dask/array/core.py", line 457, in apply_infer_dtype
    o = func(*args, **kwargs)
  File "<stdin>", line 2, in f

```

I.e., it works when I explicitly provide a `dtype`, but if not then dtype inference fails. I can work around this by providing an explicit `dtype` but thought I'd raise an issue because this didn't used to be a problem in dask 0.11. 


Discussion:

Resolving patch:
diff --git a/dask/array/core.py b/dask/array/core.py
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -457,7 +457,7 @@ def _concatenate2(arrays, axes=[]):
     return np.concatenate(arrays, axis=axes[0])
 
 
-def apply_infer_dtype(func, args, kwargs, funcname=None):
+def apply_infer_dtype(func, args, kwargs, funcname, suggest_dtype=True):
     args = [np.ones((1,) * x.ndim, dtype=x.dtype)
             if isinstance(x, Array) else x for x in args]
     try:
@@ -465,15 +465,19 @@ def apply_infer_dtype(func, args, kwargs, funcname=None):
     except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         tb = ''.join(traceback.format_tb(exc_traceback))
-        msg = ("`dtype` inference failed{0}.\n\n"
+        suggest = ("Please specify the dtype explicitly using the "
+                   "`dtype` kwarg.\n\n") if suggest_dtype else ""
+        msg = ("`dtype` inference failed in `{0}`.\n\n"
+               "{1}"
                "Original error is below:\n"
                "------------------------\n"
-               "{1}\n\n"
+               "{2}\n\n"
                "Traceback:\n"
                "---------\n"
-               "{2}"
-               ).format(" in `{0}`".format(funcname) if funcname else "",
-                        repr(e), tb)
+               "{3}").format(funcname, suggest, repr(e), tb)
+    else:
+        msg = None
+    if msg is not None:
         raise ValueError(msg)
     return o.dtype
 
@@ -2587,7 +2591,7 @@ def elemwise(op, *args, **kwargs):
         vals = [np.empty((1,) * a.ndim, dtype=a.dtype)
                 if not is_scalar_for_elemwise(a) else a
                 for a in args]
-        dt = apply_infer_dtype(op, vals, {}, 'elemwise')
+        dt = apply_infer_dtype(op, vals, {}, 'elemwise', suggest_dtype=False)
 
     name = kwargs.get('name', None) or '%s-%s' % (funcname(op),
                                                   tokenize(op, dt, *args))


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -867,6 +867,7 @@ def foo(x):
         dx.map_blocks(foo)
     except Exception as e:
         assert e.args[0].startswith("`dtype` inference failed")
+        assert "Please specify the dtype explicitly" in e.args[0]
         assert 'RuntimeError' in e.args[0]
     else:
         assert False, "Should have errored"