Repository: dask/dask

Issue:
dask array broadcast error for vector fields in structured arrays
Hi, I am seeing a value error when indexing a dask array built from a structured array with vector fields . 

An example is: 

``` python
import numpy
import dask.array as da

dtype = [('col1', ('f4', 3)), ('col2', 'f4')]
data = numpy.ones(1000, dtype=dtype)

dask_data = da.from_array(data, 100)

dask_data['col1'].compute()
```

raises the following error:

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-9-948c932886df> in <module>()
      7 dask_data = da.from_array(data, 100)
      8 
----> 9 dask_data['col1'].compute()
     10 

/Users/nhand/anaconda/envs/py35/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
     84             Extra keywords to forward to the scheduler ``get`` function.
     85         """
---> 86         return compute(self, **kwargs)[0]
     87 
     88     @classmethod

/Users/nhand/anaconda/envs/py35/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
    182     return tuple(a if not isinstance(a, Base)
    183                    else a._finalize(next(results_iter))
--> 184                    for a in args)
    185 
    186 

/Users/nhand/anaconda/envs/py35/lib/python3.5/site-packages/dask/base.py in <genexpr>(.0)
    182     return tuple(a if not isinstance(a, Base)
    183                    else a._finalize(next(results_iter))
--> 184                    for a in args)
    185 
    186 

/Users/nhand/anaconda/envs/py35/lib/python3.5/site-packages/dask/array/core.py in finalize(results)
    764     while isinstance(results2, (tuple, list)):
    765         if len(results2) > 1:
--> 766             return concatenate3(results)
    767         else:
    768             results2 = results2[0]

/Users/nhand/anaconda/envs/py35/lib/python3.5/site-packages/dask/array/core.py in concatenate3(arrays)
   3052             while arr.ndim < ndim:
   3053                 arr = arr[None, ...]
-> 3054         result[idx] = arr
   3055 
   3056     return result

ValueError: could not broadcast input array from shape (100,3) into shape (100)
```



Discussion:
cc @rainwoodman

Thanks for the bug report. ~~Looks like this is an issue with the final concatenation of results~~, looking into a fix now.

Edit: Actually it's an issue with how the graph is formed on `__getindex__` when working with structured arrays with fields that are arrays, instead of scalars.


Resolving patch:
diff --git a/dask/array/core.py b/dask/array/core.py
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -1026,15 +1026,23 @@ def __getitem__(self, index):
 
         # Field access, e.g. x['a'] or x[['a', 'b']]
         if (isinstance(index, (str, unicode)) or
-            (    isinstance(index, list)
-            and all(isinstance(i, (str, unicode)) for i in index))):
-            if self._dtype is not None and isinstance(index, (str, unicode)):
-                dt = self._dtype[index]
-            elif self._dtype is not None and isinstance(index, list):
-                dt = np.dtype([(name, self._dtype[name]) for name in index])
+                (isinstance(index, list) and
+                 all(isinstance(i, (str, unicode)) for i in index))):
+            if self._dtype is not None:
+                if isinstance(index, (str, unicode)):
+                    dt = self._dtype[index]
+                else:
+                    dt = np.dtype([(name, self._dtype[name]) for name in index])
             else:
                 dt = None
-            return elemwise(getitem, self, index, dtype=dt, name=out)
+
+            if dt is not None and dt.shape:
+                new_axis = list(range(self.ndim, self.ndim + len(dt.shape)))
+                chunks = self.chunks + tuple((i,) for i in dt.shape)
+                return self.map_blocks(getitem, index, dtype=dt.base, name=out,
+                                       chunks=chunks, new_axis=new_axis)
+            else:
+                return self.map_blocks(getitem, index, dtype=dt, name=out)
 
         # Slicing
         if not isinstance(index, tuple):


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -493,6 +493,16 @@ def test_field_access():
     assert same_keys(y[['b', 'a']], y[['b', 'a']])
 
 
+def test_field_access_with_shape():
+    dtype = [('col1', ('f4', (3, 2))), ('col2', ('f4', 3))]
+    data = np.ones((100, 50), dtype=dtype)
+    x = da.from_array(data, 10)
+    assert_eq(x['col1'], data['col1'])
+    assert_eq(x[['col1']], data[['col1']])
+    assert_eq(x['col2'], data['col2'])
+    assert_eq(x[['col1', 'col2']], data[['col1', 'col2']])
+
+
 def test_tensordot():
     x = np.arange(400).reshape((20, 20))
     a = from_array(x, chunks=(5, 4))