Repository: dask/dask

Issue:
Scalar array-like with asarray=False passed though
Using the new `asarray=False` functionality with `dask.from_array` on a scalar array-like (`shape=()`) results in the array-like itself being returned by `compute()`:

```
>>> import numpy as np
>>> import dask.array as da
>>> class MyArray:
...     shape = ()
...     dtype = np.dtype('i4')
...     def __getitem__(self, idx):
...         # This is never called by Dask
...         pass
... 
>>> a = MyArray()
>>> d = da.from_array(a, 1, asarray=False)
>>> d.compute()
<__main__.MyArray instance at 0x7f36bc726998>
```
This is preumably because `__getitem__` is never called by Dask for scalar arrays. This results in errors if you try to process the Dask object, for example reshaping it:
```
>>> d = d.reshape((1,))
>>> d.compute()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File ".../dask/base.py", line 99, in compute
    (result,) = compute(self, traverse=False, **kwargs)
  File ".../dask/base.py", line 206, in compute
    results = get(dsk, keys, **kwargs)
  File ".../dask/local.py", line 562, in get_sync
    return get_async(apply_sync, 1, dsk, keys, **kwargs)
  File ".../dask/local.py", line 529, in get_async
    fire_task()
  File ".../dask/local.py", line 504, in fire_task
    callback=queue.put)
  File ".../local.py", line 551, in apply_sync
    res = func(*args, **kwds)
  File ".../dask/local.py", line 295, in execute_task
    result = pack_exception(e, dumps)
  File ".../dask/local.py", line 290, in execute_task
    result = _execute_task(task, data)
  File ".../dask/local.py", line 270, in _execute_task
    args2 = [_execute_task(a, cache) for a in args]
  File ".../dask/local.py", line 271, in _execute_task
    return func(*args2)
  File ".../dask/utils.py", line 688, in __call__
    return getattr(obj, self.method)(*args, **kwargs)
AttributeError: MyArray instance has no attribute 'reshape'
```
The `asarray` keyword was added as part of the work to allow Dask to return masked arrays, introduced in #2301 and released in v0.15.3, though I believe v0.15.4 is more stable. I don't think it was intended to ever return the original array-like (though I could be mistaken).

@jcrist


Discussion:

Resolving patch:
diff --git a/dask/array/optimization.py b/dask/array/optimization.py
--- a/dask/array/optimization.py
+++ b/dask/array/optimization.py
@@ -11,7 +11,12 @@
 from ..utils import ensure_dict
 
 
+# All get* functions the optimizations know about
 GETTERS = (getter, getter_nofancy, getter_inline, getitem)
+# These get* functions aren't ever completely removed from the graph,
+# even if the index should be a no-op by numpy semantics. Some array-like's
+# don't completely follow semantics, making indexing always necessary.
+GETNOREMOVE = (getter, getter_nofancy)
 
 
 def optimize(dsk, keys, fuse_keys=None, fast_functions=None,
@@ -136,8 +141,8 @@ def optimize_slices(dsk):
                 a, a_index, a_lock = b, c_index, b_lock
                 a_asarray |= b_asarray
 
-            # Skip the get call if no asarray call, no lock, and nothing to do
-            if ((not a_asarray and not a_lock) and
+            # Skip the get call if not from from_array and nothing to do
+            if (get not in GETNOREMOVE and
                 ((type(a_index) is slice and not a_index.start and
                   a_index.stop is None and a_index.step is None) or
                  (type(a_index) is tuple and


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -1407,6 +1407,30 @@ def test_from_array_with_lock():
     assert_eq(e + f, x + x)
 
 
+class MyArray(object):
+    def __init__(self, x):
+        self.x = x
+        self.dtype = x.dtype
+        self.shape = x.shape
+
+    def __getitem__(self, i):
+        return self.x[i]
+
+
+def test_from_array_tasks_always_call_getter():
+    x1 = np.arange(25).reshape((5, 5))
+    x2 = np.array([[1]])
+    x3 = np.array(1)[()]
+
+    dx1a = da.from_array(MyArray(x1), chunks=(5, 5), asarray=False)
+    dx1b = da.from_array(MyArray(x1), chunks=x1.shape, asarray=False)
+    dx2 = da.from_array(MyArray(x2), chunks=1, asarray=False)
+    dx3 = da.from_array(MyArray(x3), chunks=1, asarray=False)
+
+    for res, sol in [(dx1a, x1), (dx1b, x1), (dx2, x2), (dx3, x3)]:
+        assert_eq(res, sol)
+
+
 def test_from_array_no_asarray():
 
     def assert_chunks_are_of_type(x, cls):
diff --git a/dask/array/tests/test_optimization.py b/dask/array/tests/test_optimization.py
--- a/dask/array/tests/test_optimization.py
+++ b/dask/array/tests/test_optimization.py
@@ -242,6 +242,23 @@ def test_fuse_getter_with_asarray(chunks):
     assert_eq(z, x + 1)
 
 
+@pytest.mark.parametrize('get,remove',
+                         [(getter, False), (getter_nofancy, False), (getitem, True)])
+def test_remove_no_op_slices_if_get_is_not_getter_or_getter_nofancy(get, remove):
+    # Test that no-op slices are removed as long as get is not getter or
+    # getter_nofancy. This ensures that `get` calls are always made in all
+    # tasks created by `from_array`, even after optimization
+    null = slice(0,None)
+    opts = [((get, 'x', null, False, False),
+             'x' if remove else (get, 'x', null, False, False)),
+            ((getitem, (get, 'x', null, False, False), null),
+             'x' if remove else (get, 'x', null, False, False)),
+            ((getitem, (get, 'x', (null, null), False, False), ()),
+             'x' if remove else (get, 'x', (null, null), False, False))]
+    for orig, final in opts:
+        assert optimize_slices({'a': orig}) == {'a': final}
+
+
 def test_turn_off_fusion():
     x = da.ones(10, chunks=(5,))
     y = da.sum(x + 1 + 2 + 3)