Repository: dask/dask

Issue:
Simplify graph produced by da.from_array
```
>>> dict(da.from_array(np.array([1, 2]), chunks=2).__dask_graph__())
{
    ('array-de932becc43e72c010bc91ffefe42af1', 0): (<function dask.array.core.getter(a, b, asarray=True, lock=None)>, 'array-original-de932becc43e72c010bc91ffefe42af1', (slice(0, 2, None),)),
    'array-original-de932becc43e72c010bc91ffefe42af1': array([1, 2])
}

>>> dict(da.from_array(np.array([[1, 2], [3, 4]]), chunks=(1, 2)).__dask_graph__())
{
    ('array-7cb83fc33e4505ed4b89aa67e13272f7', 0, 0): (<function dask.array.core.getter(a, b, asarray=True, lock=None)>, 'array-original-7cb83fc33e4505ed4b89aa67e13272f7', (slice(0, 1, None), slice(0, 2, None))),
    ('array-7cb83fc33e4505ed4b89aa67e13272f7', 1, 0): (<function dask.array.core.getter(a, b, asarray=True, lock=None)>, 'array-original-7cb83fc33e4505ed4b89aa67e13272f7', (slice(1, 2, None), slice(0, 2, None))),
    'array-original-7cb83fc33e4505ed4b89aa67e13272f7': array([[1, 2], [3, 4]])
}
```
The above two should really just produce:
```
>>> dict(da.from_array(np.array([1, 2]), chunks=2).__dask_graph__())
{('array-de932becc43e72c010bc91ffefe42af1', 0): array([1, 2])}

>>> dict(da.from_array(np.array([[1, 2], [3, 4]]), chunks=(1, 2)).__dask_graph__())
{
    ('array-7cb83fc33e4505ed4b89aa67e13272f7', 0, 0): (<function _operator.getitem>, 'array-original-7cb83fc33e4505ed4b89aa67e13272f7', (slice(0, 1, None))),
    ('array-7cb83fc33e4505ed4b89aa67e13272f7', 1, 0): (<function _operator.getitem>, 'array-original-7cb83fc33e4505ed4b89aa67e13272f7', (slice(1, 2, None))),
    'array-original-7cb83fc33e4505ed4b89aa67e13272f7': array([[1, 2], [3, 4]])
}
```


Discussion:
Can you explain more about why you think this?  There may be reasons for current behavior that you're not aware of.
In particular, the slicing optimizations can be pretty involved.  There are a number of important use cases that depend on slicing in particular ways.
@mrocklin sorry I don't see it. If I slice an Array with ``[:1]``, it does use operator.getitem.
I'm sure there are sophisticated use cases that warrant ``dask.array.core.getter`` - but this clearly isn't one of them?
Also I fail to understand why the trivial case of a single chunk should produce two nodes.

For getter I recommend making the change, running the test suite, and then
seeing the names of the tests that fail.

Hrm, I suspect that the two node case would fail if you were slicing from
an h5py dataset.  If objects are top-level then I think that they get
custom serialization treatment in a way that they don't if they're buried
within tasks.

All this is to say that some things that look strange are strange for a
reason, and that I generally recommend asking questions before embarking on
work.

On Mon, Jun 4, 2018 at 7:34 PM, crusaderky <notifications@github.com> wrote:

> @mrocklin <https://github.com/mrocklin> sorry I don't see it. If I slice
> an Array with [:1], it does use operator.getitem.
> I'm sure there are sophisticated use cases that warrant
> dask.array.core.getter - but this clearly isn't one of them?
> Also I fail to understand why the trivial case of a single chunk should
> produce two nodes.
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/3556#issuecomment-394533069>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszAXQjpB98w427iFHOge0PKgLed7sks5t5cQHgaJpZM4UZ7K0>
> .
>

To be clear, I mean using an h5py dataset with the distributed scheduler
with multiple processes

On Mon, Jun 4, 2018 at 8:49 PM, Matthew Rocklin <mrocklin@anaconda.com>
wrote:

> For getter I recommend making the change, running the test suite, and then
> seeing the names of the tests that fail.
>
> Hrm, I suspect that the two node case would fail if you were slicing from
> an h5py dataset.  If objects are top-level then I think that they get
> custom serialization treatment in a way that they don't if they're buried
> within tasks.
>
> All this is to say that some things that look strange are strange for a
> reason, and that I generally recommend asking questions before embarking on
> work.
>
> On Mon, Jun 4, 2018 at 7:34 PM, crusaderky <notifications@github.com>
> wrote:
>
>> @mrocklin <https://github.com/mrocklin> sorry I don't see it. If I slice
>> an Array with [:1], it does use operator.getitem.
>> I'm sure there are sophisticated use cases that warrant
>> dask.array.core.getter - but this clearly isn't one of them?
>> Also I fail to understand why the trivial case of a single chunk should
>> produce two nodes.
>>
>> —
>> You are receiving this because you were mentioned.
>> Reply to this email directly, view it on GitHub
>> <https://github.com/dask/dask/issues/3556#issuecomment-394533069>, or mute
>> the thread
>> <https://github.com/notifications/unsubscribe-auth/AASszAXQjpB98w427iFHOge0PKgLed7sks5t5cQHgaJpZM4UZ7K0>
>> .
>>
>
>


Resolving patch:
diff --git a/dask/array/core.py b/dask/array/core.py
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -2180,6 +2180,9 @@ def from_array(x, chunks, name=None, lock=False, asarray=True, fancy=True,
 
     >>> a = da.from_array(x, chunks=(1000, 1000), lock=True)  # doctest: +SKIP
     """
+    if isinstance(x, (list, tuple, memoryview) + np.ScalarType):
+        x = np.array(x)
+
     chunks = normalize_chunks(chunks, x.shape, dtype=x.dtype)
     if name in (None, True):
         token = tokenize(x, chunks)
@@ -2192,13 +2195,25 @@ def from_array(x, chunks, name=None, lock=False, asarray=True, fancy=True,
     if lock is True:
         lock = SerializableLock()
 
-    if getitem is None:
-        getitem = getter if fancy else getter_nofancy
+    # Always use the getter for h5py etc. Not using isinstance(x, np.ndarray)
+    # because np.matrix is a subclass of np.ndarray.
+    if type(x) is np.ndarray and all(len(c) == 1 for c in chunks):
+        # No slicing needed
+        dsk = {(name, ) + (0, ) * x.ndim: x}
+    else:
+        if getitem is None:
+            if type(x) is np.ndarray:
+                # simpler and cleaner, but missing all the nuances of getter
+                getitem = operator.getitem
+            elif fancy:
+                getitem = getter
+            else:
+                getitem = getter_nofancy
 
-    dsk = getem(original_name, chunks, getitem=getitem, shape=x.shape,
-                out_name=name, lock=lock, asarray=asarray,
-                dtype=x.dtype)
-    dsk[original_name] = x
+        dsk = getem(original_name, chunks, getitem=getitem, shape=x.shape,
+                    out_name=name, lock=lock, asarray=asarray,
+                    dtype=x.dtype)
+        dsk[original_name] = x
 
     return Array(dsk, name, chunks, dtype=x.dtype)
 


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -1991,39 +1991,91 @@ def __init__(self, x):
         self.x = x
         self.dtype = x.dtype
         self.shape = x.shape
+        self.ndim = len(x.shape)
 
     def __getitem__(self, i):
         return self.x[i]
 
 
-def test_from_array_tasks_always_call_getter():
-    x1 = np.arange(25).reshape((5, 5))
-    x2 = np.array([[1]])
-    x3 = np.array(1)[()]
+@pytest.mark.parametrize('x,chunks', [
+    (np.arange(25).reshape((5, 5)), (5, 5)),
+    (np.arange(25).reshape((5, 5)), -1),
+    (np.array([[1]]), 1),
+    (np.array(1), 1),
+])
+def test_from_array_tasks_always_call_getter(x, chunks):
+    dx = da.from_array(MyArray(x), chunks=chunks, asarray=False)
+    assert_eq(x, dx)
+
+
+def test_from_array_ndarray_onechunk():
+    """ndarray with a single chunk produces a minimal single key dict
+    """
+    x = np.array([[1, 2], [3, 4]])
+    dx = da.from_array(x, chunks=-1)
+    assert_eq(x, dx)
+    assert len(dx.dask) == 1
+    assert dx.dask[dx.name, 0, 0] is x
 
-    dx1a = da.from_array(MyArray(x1), chunks=(5, 5), asarray=False)
-    dx1b = da.from_array(MyArray(x1), chunks=x1.shape, asarray=False)
-    dx2 = da.from_array(MyArray(x2), chunks=1, asarray=False)
-    dx3 = da.from_array(MyArray(x3), chunks=1, asarray=False)
 
-    for res, sol in [(dx1a, x1), (dx1b, x1), (dx2, x2), (dx3, x3)]:
-        assert_eq(res, sol)
+def test_from_array_ndarray_getitem():
+    """For ndarray, don't use getter / getter_nofancy; use the cleaner
+    operator.getitem"""
+    x = np.array([[1, 2], [3, 4]])
+    dx = da.from_array(x, chunks=(1, 2))
+    assert_eq(x, dx)
+    assert dx.dask[dx.name, 0, 0][0] == operator.getitem
+
+
+@pytest.mark.parametrize(
+    'x', [[1, 2], (1, 2), memoryview(b'abc')] +
+    ([buffer(b'abc')] if sys.version_info[0] == 2 else []))  # noqa: F821
+def test_from_array_list(x):
+    """Lists, tuples, and memoryviews are automatically converted to ndarray
+    """
+    dx = da.from_array(x, chunks=-1)
+    assert_eq(np.array(x), dx)
+    assert isinstance(dx.dask[dx.name, 0], np.ndarray)
+
+    dx = da.from_array(x, chunks=1)
+    assert_eq(np.array(x), dx)
+    assert dx.dask[dx.name, 0][0] == operator.getitem
+    assert isinstance(dx.dask[dx.name.replace('array', 'array-original')],
+                      np.ndarray)
+
+
+@pytest.mark.parametrize(
+    'type_', [t for t in np.ScalarType if t not in [memoryview] +
+              ([buffer] if sys.version_info[0] == 2 else [])])  # noqa: F821
+def test_from_array_scalar(type_):
+    """Python and numpy scalars are automatically converted to ndarray
+    """
+    if type_ == np.datetime64:
+        x = np.datetime64('2000-01-01')
+    else:
+        x = type_(1)
 
+    dx = da.from_array(x, chunks=-1)
+    assert_eq(np.array(x), dx)
+    assert isinstance(dx.dask[dx.name, ], np.ndarray)
 
-def test_from_array_no_asarray():
 
-    def assert_chunks_are_of_type(x, cls):
+@pytest.mark.parametrize('asarray,cls', [
+    (True, np.ndarray),
+    (False, np.matrix),
+])
+def test_from_array_no_asarray(asarray, cls):
+
+    def assert_chunks_are_of_type(x):
         chunks = compute_as_if_collection(Array, x.dask, x.__dask_keys__())
         for c in concat(chunks):
             assert type(c) is cls
 
     x = np.matrix(np.arange(100).reshape((10, 10)))
-
-    for asarray, cls in [(True, np.ndarray), (False, np.matrix)]:
-        dx = da.from_array(x, chunks=(5, 5), asarray=asarray)
-        assert_chunks_are_of_type(dx, cls)
-        assert_chunks_are_of_type(dx[0:5], cls)
-        assert_chunks_are_of_type(dx[0:5][:, 0], cls)
+    dx = da.from_array(x, chunks=(5, 5), asarray=asarray)
+    assert_chunks_are_of_type(dx)
+    assert_chunks_are_of_type(dx[0:5])
+    assert_chunks_are_of_type(dx[0:5][:, 0])
 
 
 def test_from_array_getitem():