Repository: dask/dask

Issue:
array slicing with multiple dimensions is broken
Hi,

I can run the following code in numpy just fine:

```
import numpy as np

data = np.arange(1e6).reshape((1e6/4, 4))
print(data.shape)

data_reshaped = data[:50, 1].reshape((2, 25))
print(data_reshaped.shape)

### prints out:
# (250000, 4)
# (2, 25)
```

but in dask, it throws an exception:

```
import numpy as np
import dask.array as da

data = np.arange(1e6).reshape((1e6/4, 4))
print(data.shape) # this prints ok

dask_array = da.from_array(data, chunks=(100, ))
data_reshaped = dask_array[:50, 1].reshape((2, 25))
print(data_reshaped.compute().shape)
```

and the traceback:

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-10-0a3ca0ea6699> in <module>()
      7 dask_array = da.from_array(data, chunks=(100, ))
      8 data_reshaped = dask_array[:50, 1].reshape((2, 25))
----> 9 print(data_reshaped.compute().shape)
     10 

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
     32 
     33     def compute(self, **kwargs):
---> 34         return compute(self, **kwargs)[0]
     35 
     36     @classmethod

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
    100                 for opt, val in groups.items()])
    101     keys = [arg._keys() for arg in args]
--> 102     results = get(dsk, keys, **kwargs)
    103     return tuple(a._finalize(a, r) for a, r in zip(args, results))
    104 

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
     55     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
     56                         cache=cache, queue=queue, get_id=_thread_get_id,
---> 57                         **kwargs)
     58 
     59     return results

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/async.py in get_async(apply_async, num_workers, dsk, result, cache, queue, get_id, raise_on_exception, rerun_exceptions_locally, callbacks, **kwargs)
    481                 _execute_task(task, data)  # Re-execute locally
    482             else:
--> 483                 raise(remote_exception(res, tb))
    484         state['cache'][key] = res
    485         finish_task(dsk, key, state, results, keyorder.get)

ValueError: total size of new array must be unchanged

Traceback
---------
  File "/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/async.py", line 263, in execute_task
    result = _execute_task(task, data)
  File "/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/async.py", line 245, in _execute_task
    return func(*args2)
  File "/Users/jeremy/anaconda/lib/python3.5/site-packages/numpy/core/fromnumeric.py", line 225, in reshape
    return reshape(newshape, order=order)
```

Can anyone help me understand what is going wrong here?



Discussion:
After some debugging, it seems like the problem is in `ravel`. Test code:

```
import numpy as np
import dask.array as da

data = np.arange(1e6).reshape((1e6/4, 4))
print(data.shape)

dask_array = da.from_array(data, chunks=(100, ))
dask_array[:50, 1].ravel().compute()
```

and the result:

```
--------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-4-6500ef76d5a3> in <module>()
      6 
      7 dask_array = da.from_array(data, chunks=(100, ))
----> 8 dask_array[:50, 1].ravel().compute()
      9 

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
     32 
     33     def compute(self, **kwargs):
---> 34         return compute(self, **kwargs)[0]
     35 
     36     @classmethod

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
    101     keys = [arg._keys() for arg in args]
    102     results = get(dsk, keys, **kwargs)
--> 103     return tuple(a._finalize(a, r) for a, r in zip(args, results))
    104 
    105 

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/base.py in <genexpr>(.0)
    101     keys = [arg._keys() for arg in args]
    102     results = get(dsk, keys, **kwargs)
--> 103     return tuple(a._finalize(a, r) for a, r in zip(args, results))
    104 
    105 

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/array/core.py in finalize(arr, results)
    593 def finalize(arr, results):
    594     if arr.shape:
--> 595         return concatenate3(results)
    596     else:
    597         return unpack_singleton(results)

/Users/jeremy/anaconda/lib/python3.5/site-packages/dask/array/core.py in concatenate3(arrays)
   2482             while arr.ndim < ndim:
   2483                 arr = arr[None, ...]
-> 2484         result[idx] = arr
   2485 
   2486     return result

ValueError: could not broadcast input array from shape (50,4) into shape (50)
```

And turns out, the problem is neither, because the following code has the same problem:

```
import numpy as np
import dask.array as da

data = np.arange(1e6).reshape((1e6/4, 4))
print(data.shape)

dask_array = da.from_array(data, chunks=(100, ))
dask_array[:50, 1].compute()
```

More debugging shows that the resulting graph generated by the above code ignores the second dimension. So the problem appears to be in the slicing.

```
(PdB) dsk
{('getitem-53735434f2afd704d21f48b61e1a663c', 0): (<function getarray at 0x107a9ebf8>, array([[  0.00000000e+00,   1.00000000e+00,   2.00000000e+00,
          3.00000000e+00],
       [  4.00000000e+00,   5.00000000e+00,   6.00000000e+00,
          7.00000000e+00],
       [  8.00000000e+00,   9.00000000e+00,   1.00000000e+01,
          1.10000000e+01],
       ..., 
       [  9.99988000e+05,   9.99989000e+05,   9.99990000e+05,
          9.99991000e+05],
       [  9.99992000e+05,   9.99993000e+05,   9.99994000e+05,
          9.99995000e+05],
       [  9.99996000e+05,   9.99997000e+05,   9.99998000e+05,
          9.99999000e+05]]), (slice(0, 50, None),))}
```

I'd be happy to try to patch this myself, but I'm still struggling to understand the slice generation logic and the exact meaning of the function arguments.


Resolving patch:
diff --git a/dask/array/core.py b/dask/array/core.py
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -1411,6 +1411,9 @@ def from_array(x, chunks, name=None, lock=False):
     >>> a = da.from_array(x, chunks=(1000, 1000), lock=True)  # doctest: +SKIP
     """
     chunks = normalize_chunks(chunks, x.shape)
+    if len(chunks) != len(x.shape):
+        raise ValueError("Input array has %d dimensions but the supplied "
+                "chunks has only %d dimensions" % (len(x.shape), len(chunks)))
     name = name or 'from-array-' + tokenize(x, chunks)
     dsk = getem(name, chunks)
     if lock is True:


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -1721,3 +1721,9 @@ def test_broadcast_chunks():
     b = ((3, 3,), (5, 5),)
     with pytest.raises(ValueError):
         broadcast_chunks(a, b)
+
+
+def test_chunks_error():
+    x = np.ones((10, 10))
+    with pytest.raises(ValueError):
+        da.from_array(x, chunks=(5,))