Repository: dask/dask

Issue:
Indexing error with boolean array
This was reported to me in a side channel

```python
In [1]: import numpy as np

In [2]: import dask.array as da

In [3]: da.asarray(np.ones(1))[np.array([True], dtype=bool)]
   ...: 
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-3-486aaf80dd68> in <module>()
----> 1 da.asarray(np.ones(1))[np.array([True], dtype=bool)]

~/workspace/dask/dask/array/core.py in __getitem__(self, index)
   1326 
   1327         from .slicing import normalize_index, slice_with_dask_array
-> 1328         index2 = normalize_index(index, self.shape)
   1329 
   1330         if any(isinstance(i, Array) for i in index2):

~/workspace/dask/dask/array/slicing.py in normalize_index(idx, shape)
    781     for i, d in zip(idx, none_shape):
    782         if d is not None:
--> 783             check_index(i, d)
    784     idx = tuple(map(sanitize_index, idx))
    785     idx = tuple(map(normalize_slice, idx, none_shape))

~/workspace/dask/dask/array/slicing.py in check_index(ind, dimension)
    824         x = np.asanyarray(ind)
    825         if (x >= dimension).any() or (x < -dimension).any():
--> 826             raise IndexError("Index out of bounds %s" % dimension)
    827     elif isinstance(ind, slice):
    828         return

IndexError: Index out of bounds 1

In [4]: np.asarray(np.ones(1))[np.array([True], dtype=bool)]
   ...: 
   ...: 
Out[4]: array([1.])
```


Discussion:
So I looked into this a bit and it seems fairly easy and the cause is actually in the stacktrace - the `check_index` function compares `x` which is the boolean array with the dimension which is a number, so the `x` array is treated as a 0/1 array. This caused the error above since `True` has integer value 1 which equals to the length of the array.

This bug also has another manifestation. Since dask just compares the elements to the array length it missed the constraint that the boolean indexing array should have length `dimension` (see ["Boolean arrays must be of the same shape as the initial dimensions of the array being indexed"](https://docs.scipy.org/doc/numpy/user/basics.indexing.html#boolean-or-mask-index-arrays). E.g.,

    >>> da.asarray(np.arange(5))[np.array([True, True], dtype=bool)].compute()
    array([0, 1])
    >>> np.asarray(np.arange(5))[np.array([True, True], dtype=bool)]
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
    IndexError: boolean index did not match indexed array along dimension 0; dimension is 5 but corresponding boolean dimension is 2

This should be pretty easy to fix - I can send out a pull request this week if the above sounds reasonable.

A fix would be welcome

On Tue, Jul 3, 2018 at 11:55 PM, WZY <notifications@github.com> wrote:

> So I looked into this a bit and it seems fairly easy and the cause is
> actually in the stacktrace - the check_index function compares x which is
> the boolean array with the dimension which is a number, so the x array is
> treated as a 0/1 array. This caused the error above since True has
> integer value 1 which equals to the length of the array.
>
> This bug also has another manifestation. Since dask just compares the
> elements to the array length it missed the constraint that the boolean
> indexing array should have length dimension (see "Boolean arrays must be
> of the same shape as the initial dimensions of the array being indexed"
> <https://docs.scipy.org/doc/numpy/user/basics.indexing.html#boolean-or-mask-index-arrays>.
> E.g.,
>
> >>> da.asarray(np.arange(5))[np.array([True, True], dtype=bool)].compute()
> array([0, 1])
> >>> np.asarray(np.arange(5))[np.array([True, True], dtype=bool)]
> Traceback (most recent call last):
>   File "<stdin>", line 1, in <module>
> IndexError: boolean index did not match indexed array along dimension 0; dimension is 5 but corresponding boolean dimension is 2
>
> This should be pretty easy to fix - I can send out a pull request this
> week if the above sounds reasonable.
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/3706#issuecomment-402355895>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszMbPyZ4RGqDEEFRI55uMbWfGVmRTks5uDDytgaJpZM4U_LT6>
> .
>


Resolving patch:
diff --git a/dask/array/slicing.py b/dask/array/slicing.py
--- a/dask/array/slicing.py
+++ b/dask/array/slicing.py
@@ -816,13 +816,28 @@ def check_index(ind, dimension):
     IndexError: Index out of bounds 5
 
     >>> check_index(slice(0, 3), 5)
+
+    >>> check_index([True], 1)
+    >>> check_index([True, True], 3)
+    Traceback (most recent call last):
+    ...
+    IndexError: Boolean array length 2 doesn't equal dimension 3
+    >>> check_index([True, True, True], 1)
+    Traceback (most recent call last):
+    ...
+    IndexError: Boolean array length 3 doesn't equal dimension 1
     """
     # unknown dimension, assumed to be in bounds
     if np.isnan(dimension):
         return
     elif isinstance(ind, (list, np.ndarray)):
         x = np.asanyarray(ind)
-        if (x >= dimension).any() or (x < -dimension).any():
+        if x.dtype == bool:
+            if x.size != dimension:
+                raise IndexError(
+                    "Boolean array length %s doesn't equal dimension %s" %
+                    (x.size, dimension))
+        elif (x >= dimension).any() or (x < -dimension).any():
             raise IndexError("Index out of bounds %s" % dimension)
     elif isinstance(ind, slice):
         return


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -1239,7 +1239,7 @@ def test_slicing_with_ndarray():
     assert_eq(d[np.arange(8)], x)
     assert_eq(d[np.ones(8, dtype=bool)], x)
     assert_eq(d[np.array([1])], x[[1]])
-    assert_eq(d[np.array([True])], x[[0]])
+    assert_eq(d[np.array([True, False, True] + [False] * 5)], x[[0, 2]])
 
 
 def test_dtype():
diff --git a/dask/array/tests/test_slicing.py b/dask/array/tests/test_slicing.py
--- a/dask/array/tests/test_slicing.py
+++ b/dask/array/tests/test_slicing.py
@@ -311,7 +311,7 @@ def test_slicing_with_numpy_arrays():
     np.testing.assert_equal(a, b)
 
     i = [False, True, True, False, False,
-         False, False, False, False, True, False]
+         False, False, False, False, True]
     index = (i, slice(None, None, None))
     index = normalize_index(index, (10, 10))
     c, bd3 = slice_array('y', 'x', ((3, 3, 3, 1), (3, 3, 3, 1)), index)
@@ -413,6 +413,34 @@ def test_multiple_list_slicing():
     assert_eq(x[:, [0, 1, 2]][[0, 1]], a[:, [0, 1, 2]][[0, 1]])
 
 
+@pytest.mark.skipif(np.__version__ < '1.13.0',
+                    reason='boolean lists are not treated as boolean indexes')
+def test_boolean_list_slicing():
+    with pytest.raises(IndexError):
+        da.asarray(range(2))[[True]]
+    with pytest.raises(IndexError):
+        da.asarray(range(2))[[False, False, False]]
+    x = np.arange(5)
+    ind = [True, False, False, False, True]
+    assert_eq(da.asarray(x)[ind], x[ind])
+    # https://github.com/dask/dask/issues/3706
+    ind = [True]
+    assert_eq(da.asarray([0])[ind], np.arange(1)[ind])
+
+
+def test_boolean_numpy_array_slicing():
+    with pytest.raises(IndexError):
+        da.asarray(range(2))[np.array([True])]
+    with pytest.raises(IndexError):
+        da.asarray(range(2))[np.array([False, False, False])]
+    x = np.arange(5)
+    ind = np.array([True, False, False, False, True])
+    assert_eq(da.asarray(x)[ind], x[ind])
+    # https://github.com/dask/dask/issues/3706
+    ind = np.array([True])
+    assert_eq(da.asarray([0])[ind], np.arange(1)[ind])
+
+
 def test_empty_list():
     x = np.ones((5, 5, 5), dtype='i4')
     dx = da.from_array(x, chunks=2)