Repository: dask/dask

Issue:
dask.array regression involving arithmetic with xarray objects
This code worked with dask v0.7.6, but failed when I updated to v0.9.0:

```
In [1]: import numpy as np

In [2]: import xarray as xr

In [3]: import dask.array as da

In [4]: x = xr.core.indexing.LazilyIndexedArray(np.zeros((3, 3)))

In [5]: y = da.from_array(x, (2, 2))

In [6]: (y[:2, :2] * y[:2, :2]).compute()
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-6-82666a5a7d29> in <module>()
----> 1 (y[:2, :2] * y[:2, :2]).compute()

/Users/shoyer/conda/envs/xarray-dev/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
     35
     36     def compute(self, **kwargs):
---> 37         return compute(self, **kwargs)[0]
     38
     39     @classmethod

/Users/shoyer/conda/envs/xarray-dev/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
    108                 for opt, val in groups.items()])
    109     keys = [var._keys() for var in variables]
--> 110     results = get(dsk, keys, **kwargs)
    111
    112     results_iter = iter(results)

/Users/shoyer/conda/envs/xarray-dev/lib/python3.5/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
     55     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
     56                         cache=cache, queue=queue, get_id=_thread_get_id,
---> 57                         **kwargs)
     58
     59     return results

/Users/shoyer/conda/envs/xarray-dev/lib/python3.5/site-packages/dask/async.py in get_async(apply_async, num_workers, dsk, result, cache, queue, get_id, raise_on_exception, rerun_exceptions_locally, callbacks, **kwargs)
    486                 _execute_task(task, data)  # Re-execute locally
    487             else:
--> 488                 raise(remote_exception(res, tb))
    489         state['cache'][key] = res
    490         finish_task(dsk, key, state, results, keyorder.get)

TypeError: unsupported operand type(s) for *: 'LazilyIndexedArray' and 'LazilyIndexedArray'

Traceback
---------
  File "/Users/shoyer/conda/envs/xarray-dev/lib/python3.5/site-packages/dask/async.py", line 267, in execute_task
    result = _execute_task(task, data)
  File "/Users/shoyer/conda/envs/xarray-dev/lib/python3.5/site-packages/dask/async.py", line 249, in _execute_task
    return func(*args2)
```

LazilyIndexedArray is an object that satisfies some of the usual duck-array API (e.g. shape, dtype, `__getitem__`) and that can be coerced into a NumPy array:
https://github.com/pydata/xarray/blob/v0.7.2/xarray/core/indexing.py#L272

I _think_ it should be valid into to `da.from_array` -- this certainly worked in old versions of dask.



Discussion:
This was a bug in our optimizations refactor. PR coming soon.


Resolving patch:
diff --git a/dask/array/optimization.py b/dask/array/optimization.py
--- a/dask/array/optimization.py
+++ b/dask/array/optimization.py
@@ -3,12 +3,10 @@
 from operator import getitem
 
 import numpy as np
-from toolz import valmap, partial
 
 from .core import getarray
 from ..core import flatten
-from ..optimize import cull, fuse, dealias, inline_functions
-from ..rewrite import RuleSet, RewriteRule
+from ..optimize import cull, fuse, inline_functions
 
 
 def optimize(dsk, keys, **kwargs):
@@ -20,12 +18,12 @@ def optimize(dsk, keys, **kwargs):
     """
     keys = list(flatten(keys))
     fast_functions = kwargs.get('fast_functions',
-                             set([getarray, np.transpose]))
+                                set([getarray, np.transpose]))
     dsk2, dependencies = cull(dsk, keys)
     dsk4, dependencies = fuse(dsk2, keys, dependencies)
     dsk5 = optimize_slices(dsk4)
     dsk6 = inline_functions(dsk5, keys, fast_functions=fast_functions,
-            dependencies=dependencies)
+                            dependencies=dependencies)
     return dsk6
 
 
@@ -38,34 +36,39 @@ def optimize_slices(dsk):
     See also:
         fuse_slice_dict
     """
+    getters = (getarray, getitem)
     dsk = dsk.copy()
     for k, v in dsk.items():
         if type(v) is tuple:
-            if v[0] is getitem or v[0] is getarray:
+            if v[0] in getters:
                 try:
                     func, a, a_index = v
+                    use_getarray = func is getarray
                 except ValueError:  # has four elements, includes a lock
                     continue
-                while type(a) is tuple and (a[0] is getitem or a[0] is getarray):
+                while type(a) is tuple and a[0] in getters:
                     try:
-                        _, b, b_index = a
+                        f2, b, b_index = a
+                        use_getarray |= f2 is getarray
                     except ValueError:  # has four elements, includes a lock
                         break
                     if (type(a_index) is tuple) != (type(b_index) is tuple):
                         break
                     if ((type(a_index) is tuple) and
-                        (len(a_index) != len(b_index)) and
-                        any(i is None for i in b_index + a_index)):
+                            (len(a_index) != len(b_index)) and
+                            any(i is None for i in b_index + a_index)):
                         break
                     try:
                         c_index = fuse_slice(b_index, a_index)
                     except NotImplementedError:
                         break
                     (a, a_index) = (b, c_index)
-                if (type(a_index) is slice and
-                    not a_index.start and
-                    a_index.stop is None and
-                    a_index.step is None):
+                if use_getarray:
+                    dsk[k] = (getarray, a, a_index)
+                elif (type(a_index) is slice and
+                      not a_index.start and
+                      a_index.stop is None and
+                      a_index.step is None):
                     dsk[k] = a
                 elif type(a_index) is tuple and all(type(s) is slice and
                                                     not s.start and
@@ -74,7 +77,7 @@ def optimize_slices(dsk):
                                                     for s in a_index):
                     dsk[k] = a
                 else:
-                    dsk[k] = (func, a, a_index)
+                    dsk[k] = (getitem, a, a_index)
     return dsk
 
 
@@ -167,7 +170,7 @@ def fuse_slice(a, b):
     if isinstance(a, tuple) and isinstance(b, tuple):
 
         if (any(isinstance(item, list) for item in a) and
-            any(isinstance(item, list) for item in b)):
+                any(isinstance(item, list) for item in b)):
             raise NotImplementedError("Can't handle multiple list indexing")
 
         j = 0


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -939,10 +939,7 @@ def test_store_locks():
 
 
 def test_to_hdf5():
-    try:
-        import h5py
-    except ImportError:
-        return
+    h5py = pytest.importorskip('h5py')
     x = da.ones((4, 4), chunks=(2, 2))
     y = da.ones(4, chunks=2, dtype='i4')
 
@@ -1296,6 +1293,17 @@ def test_from_array_with_lock():
     assert_eq(e + f, x + x)
 
 
+def test_from_array_slicing_results_in_ndarray():
+    x = np.matrix(np.arange(100).reshape((10, 10)))
+    dx = da.from_array(x, chunks=(5, 5))
+    s1 = dx[0:5]
+    assert type(dx[0:5].compute()) == np.ndarray
+    s2 = s1[0:3]
+    assert type(s2.compute()) == np.ndarray
+    s3 = s2[:, 0]
+    assert type(s3.compute()) == np.ndarray
+
+
 def test_from_func():
     x = np.arange(10)
     f = lambda n: n * x
@@ -1550,10 +1558,7 @@ def test_long_slice():
 
 
 def test_h5py_newaxis():
-    try:
-        import h5py
-    except ImportError:
-        return
+    h5py = pytest.importorskip('h5py')
 
     with tmpfile('h5') as fn:
         with h5py.File(fn) as f:
diff --git a/dask/array/tests/test_optimization.py b/dask/array/tests/test_optimization.py
--- a/dask/array/tests/test_optimization.py
+++ b/dask/array/tests/test_optimization.py
@@ -16,14 +16,14 @@ def test_fuse_getitem():
 
              ((getitem, (getarray, 'x', (slice(1000, 2000), slice(100, 200))),
                         (slice(15, 20), slice(50, 60))),
-              (getitem, 'x', (slice(1015, 1020), slice(150, 160)))),
+              (getarray, 'x', (slice(1015, 1020), slice(150, 160)))),
 
              ((getarray, (getarray, 'x', slice(1000, 2000)), 10),
               (getarray, 'x', 1010)),
 
              ((getitem, (getarray, 'x', (slice(1000, 2000), 10)),
                         (slice(15, 20),)),
-              (getitem, 'x', (slice(1015, 1020), 10))),
+              (getarray, 'x', (slice(1015, 1020), 10))),
 
              ((getarray, (getarray, 'x', (10, slice(1000, 2000))),
                         (slice(15, 20),)),
@@ -73,9 +73,9 @@ def test_optimize_slicing():
     assert result == expected
 
     # protect output keys
-    expected = {'c': (range, 10),
+    expected = {'c': (getarray, (range, 10), (slice(0, None, None),)),
                 'd': (getarray, 'c', (slice(0, 5, None),)),
-                'e': 'd'}
+                'e': (getarray, 'd', (slice(None, None, None),))}
     result = optimize_slices(fuse(dsk, ['c', 'd', 'e'])[0])
 
     assert result == expected