Repository: dask/dask

Issue:
IndexError: tuple index out of range in optimization.py
Hi, I apologize but unfortunately I can't reproduce this issue without copy and pasting a lot of code. I was hoping that perhaps this would look familiar to somebody here and they might be able to point me toward a workaround.

If it looks like it's a more serious issue, I can try to isolate code to come up with a self-contained example.

```
     62 
     63         print("old_data.shape", old_data.shape, values_dset.shape)
---> 64         da.store(old_data, values_dset)
     65         # f_new['values_' + str(z)][:] = old_data
     66 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in store(sources, targets, lock, regions, compute, return_stored, **kwargs)
    959 
    960         if compute:
--> 961             result.compute(**kwargs)
    962             return None
    963         else:

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(self, **kwargs)
    154         dask.base.compute
    155         """
--> 156         (result,) = compute(self, traverse=False, **kwargs)
    157         return result
    158 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
    400     keys = [x.__dask_keys__() for x in collections]
    401     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 402     results = schedule(dsk, keys, **kwargs)
    403     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    404 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
     73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
     74                         cache=cache, get_id=_thread_get_id,
---> 75                         pack_exception=pack_exception, **kwargs)
     76 
     77     # Cleanup pools associated to dead threads

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    519                         _execute_task(task, data)  # Re-execute locally
    520                     else:
--> 521                         raise_exception(exc, tb)
    522                 res, worker_id = loads(res_info)
    523                 state['cache'][key] = res

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/compatibility.py in reraise(exc, tb)
     67         if exc.__traceback__ is not tb:
     68             raise exc.with_traceback(tb)
---> 69         raise exc
     70 
     71 else:

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    288     try:
    289         task, data = loads(task_info)
--> 290         result = _execute_task(task, data)
    291         id = get_id()
    292         result = dumps((result, id))

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/local.py in _execute_task(arg, cache, dsk)
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in getter_inline(a, b, asarray, lock)
    118     This inlining can be relevant to operations when running off of disk.
    119     """
--> 120     return getter(a, b, asarray=asarray, lock=lock)
    121 
    122 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in getter(a, b, asarray, lock)
     83         c = a[b]
     84         if asarray:
---> 85             c = np.asarray(c)
     86     finally:
     87         if lock:

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/numpy/core/numeric.py in asarray(a, dtype, order)
    499 
    500     """
--> 501     return array(a, dtype, copy=False, order=order)
    502 
    503 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in __array__(self, dtype, **kwargs)
   1207 
   1208     def __array__(self, dtype=None, **kwargs):
-> 1209         x = self.compute()
   1210         if dtype and x.dtype != dtype:
   1211             x = x.astype(dtype)

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(self, **kwargs)
    154         dask.base.compute
    155         """
--> 156         (result,) = compute(self, traverse=False, **kwargs)
    157         return result
    158 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
    397                              collections=collections)
    398 
--> 399     dsk = collections_to_dsk(collections, optimize_graph, **kwargs)
    400     keys = [x.__dask_keys__() for x in collections]
    401     postcomputes = [x.__dask_postcompute__() for x in collections]

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in collections_to_dsk(collections, optimize_graph, **kwargs)
    192 
    193         dsk = merge(*(opt(dsk, keys, **kwargs)
--> 194                       for opt, (dsk, keys) in groups.items()))
    195     else:
    196         dsk, _ = _extract_graph_and_keys(collections)

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in <genexpr>(.0)
    192 
    193         dsk = merge(*(opt(dsk, keys, **kwargs)
--> 194                       for opt, (dsk, keys) in groups.items()))
    195     else:
    196         dsk, _ = _extract_graph_and_keys(collections)

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/optimization.py in optimize(dsk, keys, fuse_keys, fast_functions, inline_functions_fast_functions, rename_fused_keys, **kwargs)
     44     else:
     45         dsk4 = dsk3
---> 46     dsk5 = optimize_slices(dsk4)
     47 
     48     return dsk5

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/optimization.py in optimize_slices(dsk)
    132                     break
    133                 try:
--> 134                     c_index = fuse_slice(b_index, a_index)
    135                     # rely on fact that nested gets never decrease in
    136                     # strictness e.g. `(getter_nofancy, (getter, ...))` never

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/optimization.py in fuse_slice(a, b)
    280                 result.append(a[i])
    281                 continue
--> 282             while b[j] is None:  # insert any Nones on the rhs
    283                 result.append(None)
    284                 j += 1

IndexError: tuple index out of range
```

Python: 3.6.2
Dask: 0.18.1



Discussion:
Thanks for the error report @pkerpedjiev 

The next step to solving this is to find a reproducible example.  If you have time to help with that process then that would definitely accelerate things.
Thanks for the quick response! This reproduces the error on two separate computers. It works on dask `0.15.0` but fails on `0.18.1` and `0.19.0`.

```
import h5py
import numpy as np
import math

data_size = 6645370*2
chunk_size = 2**21

import dask
import dask.array as da
print(dask.__version__)

with h5py.File('/tmp/blah.h5', 'w') as f:
    ds = f.create_dataset('data', (data_size,), dtype='f')
    ds[:] = np.random.random((data_size,))
    
with h5py.File('/tmp/blah.h5', 'r') as fin:
    old_data = da.from_array(fin['data'], chunks=(chunk_size,))
    
    with h5py.File('/tmp/blah1.h5', 'w') as fout:
        z = 0
        while data_size > 1024:
            values_dset = fout.require_dataset('values_' + str(z), 
                        (len(old_data),), 
                         dtype='f', compression='gzip' )
            
            da.store(old_data, values_dset)
            
            zoom_factor = 2
            if len(old_data) % zoom_factor != 0:
                old_data = da.concatenate(
                (old_data, [old_data[-1]] * ( zoom_factor - len(old_data) % zoom_factor )))
            
            old_data = old_data.rechunk((chunk_size,))
            old_data = da.coarsen(np.sum, old_data, {0: 2})
            
            data_size = len(old_data)
            z += 1
            
            print("data_size:", data_size, "chunks:", chunk_size)
```
Do you need h5py to make this work, or would it also work with empty numpy
arrays?

On Tue, Sep 4, 2018 at 9:55 AM, Peter Kerpedjiev <notifications@github.com>
wrote:

> This reproduces the error on two separate computers. It works on dask
> 0.15.0 but fails on 0.18.1 and 0.19.0.
>
> import h5py
> import numpy as np
> import math
>
> data_size = 6645370*2
> chunk_size = 2**21
>
> import dask
> import dask.array as da
> print(dask.__version__)
>
> with h5py.File('/tmp/blah.h5', 'w') as f:
>     ds = f.create_dataset('data', (data_size,), dtype='f')
>     ds[:] = np.random.random((data_size,))
>
> with h5py.File('/tmp/blah.h5', 'r') as fin:
>     old_data = da.from_array(fin['data'], chunks=(chunk_size,))
>
>     with h5py.File('/tmp/blah1.h5', 'w') as fout:
>         z = 0
>         while data_size > 1024:
>             values_dset = fout.require_dataset('values_' + str(z),
>                         (len(old_data),),
>                          dtype='f', compression='gzip' )
>
>             da.store(old_data, values_dset)
>
>             zoom_factor = 2
>             if len(old_data) % zoom_factor != 0:
>                 old_data = da.concatenate(
>                 (old_data, [old_data[-1]] * ( zoom_factor - len(old_data) % zoom_factor )))
>
>             old_data = old_data.rechunk((chunk_size,))
>             old_data = da.coarsen(np.sum, old_data, {0: 2})
>
>             data_size = len(old_data)
>             z += 1
>
>             print("data_size:", data_size, "chunks:", chunk_size)
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/3937#issuecomment-418376007>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszIqm05h_BdPwD2gLFccIxKzKlSDzks5uXoZLgaJpZM4WYIT1>
> .
>

For my use case I need h5py but I can also reproduce the error using regular numpy arrays:

```
data_size = 6645370*2
chunk_size = 2**21

old_data = da.from_array(np.random.random((data_size,)), (chunk_size,))

z = 0
while data_size > 1024:
    x = old_data.compute()
    
    zoom_factor = 2
    if len(old_data) % zoom_factor != 0:
        old_data = da.concatenate(
        (old_data, [old_data[-1]] * ( zoom_factor - len(old_data) % zoom_factor )))

    old_data = old_data.rechunk((chunk_size,))
    old_data = da.coarsen(np.sum, old_data, {0: 2})

    data_size = len(old_data)
    z += 1

    print("data_size:", data_size, "chunks:", chunk_size)
```
The simpler you can make this example the better.  It really really helps
to have simple examples to work from.

Is there something special about that size in particular?  Is the while
loop necessary? I suspect that some of these are important for the problem
and some of them are specific to your code.  It takes maintainers a lot of
time to figure out which is which.  If you're able to reduce this down to
as small an example as you can that would really help.

http://matthewrocklin.com/blog/work/2016/08/25/supporting-users

On Tue, Sep 4, 2018 at 10:04 AM, Peter Kerpedjiev <notifications@github.com>
wrote:

> For my use case I need h5py but I can also reproduce the error using
> regular numpy arrays:
>
> data_size = 6645370*2
> chunk_size = 2**21
>
> old_data = da.from_array(np.random.random((data_size,)), (chunk_size,))
>
> z = 0
> while data_size > 1024:
>     x = old_data.compute()
>
>     zoom_factor = 2
>     if len(old_data) % zoom_factor != 0:
>         old_data = da.concatenate(
>         (old_data, [old_data[-1]] * ( zoom_factor - len(old_data) % zoom_factor )))
>
>     old_data = old_data.rechunk((chunk_size,))
>     old_data = da.coarsen(np.sum, old_data, {0: 2})
>
>     data_size = len(old_data)
>     z += 1
>
>     print("data_size:", data_size, "chunks:", chunk_size)
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/3937#issuecomment-418379186>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszE34wBLZ7D9-Wv4IA9EWjtKDzTcBks5uXohkgaJpZM4WYIT1>
> .
>

Sorry, I meant to copy a link to this blogpost on minimal bug reports:
http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports

On Tue, Sep 4, 2018 at 10:15 AM, Matthew Rocklin <mrocklin@anaconda.com>
wrote:

> The simpler you can make this example the better.  It really really helps
> to have simple examples to work from.
>
> Is there something special about that size in particular?  Is the while
> loop necessary? I suspect that some of these are important for the problem
> and some of them are specific to your code.  It takes maintainers a lot of
> time to figure out which is which.  If you're able to reduce this down to
> as small an example as you can that would really help.
>
> http://matthewrocklin.com/blog/work/2016/08/25/supporting-users
>
> On Tue, Sep 4, 2018 at 10:04 AM, Peter Kerpedjiev <
> notifications@github.com> wrote:
>
>> For my use case I need h5py but I can also reproduce the error using
>> regular numpy arrays:
>>
>> data_size = 6645370*2
>> chunk_size = 2**21
>>
>> old_data = da.from_array(np.random.random((data_size,)), (chunk_size,))
>>
>> z = 0
>> while data_size > 1024:
>>     x = old_data.compute()
>>
>>     zoom_factor = 2
>>     if len(old_data) % zoom_factor != 0:
>>         old_data = da.concatenate(
>>         (old_data, [old_data[-1]] * ( zoom_factor - len(old_data) % zoom_factor )))
>>
>>     old_data = old_data.rechunk((chunk_size,))
>>     old_data = da.coarsen(np.sum, old_data, {0: 2})
>>
>>     data_size = len(old_data)
>>     z += 1
>>
>>     print("data_size:", data_size, "chunks:", chunk_size)
>>
>> —
>> You are receiving this because you commented.
>> Reply to this email directly, view it on GitHub
>> <https://github.com/dask/dask/issues/3937#issuecomment-418379186>, or mute
>> the thread
>> <https://github.com/notifications/unsubscribe-auth/AASszE34wBLZ7D9-Wv4IA9EWjtKDzTcBks5uXohkgaJpZM4WYIT1>
>> .
>>
>
>

Sure. This is about as simple as I can get it:

```
data_size = 3322683
chunk_size = 2**20

old_data = da.from_array(np.random.random((data_size,)), (chunk_size,))
old_data = da.concatenate((old_data, [old_data[-1]]))

old_data = old_data.rechunk((chunk_size,))
old_data = da.coarsen(np.sum, old_data, {0: 2})

old_data.compute()
```

Both the data size and chunk size seem to be important. Changing them to certain values (i.e. `chunk_size=int(1e8)` or `data_size=322683`) can make the error disappear. The `concatenate` statement is necessary. Starting from `data_size=3322684` does not produce an error.
I just ran `git bisect` on the issue. Here's the commit that introduced this error:

```
(cenv3) peter@mbi-cw-l10381:~/projects/dask [(no branch, bisect started on master)]$ git bisect bad
7c419580037f552befc2650cb13967dd6bdef86a is the first bad commit
commit 7c419580037f552befc2650cb13967dd6bdef86a
Author: Yu Feng <rainwoodman@gmail.com>
Date:   Wed May 9 03:03:48 2018 -0700

    Faster slice_1d in dask.array (#3479)

    This uses numpy and binary search to accelerate slicing performance
    when there are many chunks along a dimension

    See also benchmarks in https://github.com/dask/dask-benchmarks/pull/15

:040000 040000 c20f6895a1dd93a7ab257253496cba6149e07674 195af40fac18ce1112aa6f3f8fd9f49138ff83fb M      dask
```
cc @rainwoodman

On Tue, Sep 4, 2018 at 10:36 AM, Peter Kerpedjiev <notifications@github.com>
wrote:

> I just ran git bisect on the issue. Here's the commit that introduced
> this error:
>
> (cenv3) peter@mbi-cw-l10381:~/projects/dask [(no branch, bisect started on master)]$ git bisect bad
> 7c419580037f552befc2650cb13967dd6bdef86a is the first bad commit
> commit 7c419580037f552befc2650cb13967dd6bdef86a
> Author: Yu Feng <rainwoodman@gmail.com>
> Date:   Wed May 9 03:03:48 2018 -0700
>
>     Faster slice_1d in dask.array (#3479)
>
>     This uses numpy and binary search to accelerate slicing performance
>     when there are many chunks along a dimension
>
>     See also benchmarks in https://github.com/dask/dask-benchmarks/pull/15
>
> :040000 040000 c20f6895a1dd93a7ab257253496cba6149e07674 195af40fac18ce1112aa6f3f8fd9f49138ff83fb M      dask
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/3937#issuecomment-418390864>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszIpCayPAI0v5SLZeYPz3KtCbZ2p_ks5uXo_XgaJpZM4WYIT1>
> .
>

Interesting. I will work out a fix today.
Thanks @rainwoodman

On Tue, Sep 4, 2018 at 3:05 PM, Yu Feng <notifications@github.com> wrote:

> Interesting. I will work out a fix today.
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/3937#issuecomment-418482308>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszDm2F1fR4PuKukHN02JI0jwmjI66ks5uXs7ngaJpZM4WYIT1>
> .
>


Resolving patch:
diff --git a/dask/array/slicing.py b/dask/array/slicing.py
--- a/dask/array/slicing.py
+++ b/dask/array/slicing.py
@@ -391,7 +391,7 @@ def _slice_1d(dim_shape, lengths, index):
             ind = index - chunk_boundaries[i - 1]
         else:
             ind = index
-        return {i: ind}
+        return {int(i): int(ind)}
 
     assert isinstance(index, slice)
 


Test patch:
diff --git a/dask/array/tests/test_optimization.py b/dask/array/tests/test_optimization.py
--- a/dask/array/tests/test_optimization.py
+++ b/dask/array/tests/test_optimization.py
@@ -273,3 +273,15 @@ def test_turn_off_fusion():
 
     assert dask.get(a, y.__dask_keys__()) == dask.get(b, y.__dask_keys__())
     assert len(a) < len(b)
+
+
+def test_gh3937():
+    # test for github issue #3937
+    x = da.from_array([1, 2, 3.], (2,))
+    x = da.concatenate((x, [x[-1]]))
+    y = x.rechunk((2,))
+    # This will produce Integral type indices that are not ints (np.int64), failing
+    # the optimizer
+    y = da.coarsen(np.sum, y, {0: 2})
+    # How to trigger the optimizer explicitly?
+    y.compute()