Repository: dask/dask

Issue:
IndexError: tuple index out of range in optimization.py
Hi, I apologize but unfortunately I can't reproduce this issue without copy and pasting a lot of code. I was hoping that perhaps this would look familiar to somebody here and they might be able to point me toward a workaround.

If it looks like it's a more serious issue, I can try to isolate code to come up with a self-contained example.

```
     62 
     63         print("old_data.shape", old_data.shape, values_dset.shape)
---> 64         da.store(old_data, values_dset)
     65         # f_new['values_' + str(z)][:] = old_data
     66 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in store(sources, targets, lock, regions, compute, return_stored, **kwargs)
    959 
    960         if compute:
--> 961             result.compute(**kwargs)
    962             return None
    963         else:

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(self, **kwargs)
    154         dask.base.compute
    155         """
--> 156         (result,) = compute(self, traverse=False, **kwargs)
    157         return result
    158 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
    400     keys = [x.__dask_keys__() for x in collections]
    401     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 402     results = schedule(dsk, keys, **kwargs)
    403     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    404 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
     73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
     74                         cache=cache, get_id=_thread_get_id,
---> 75                         pack_exception=pack_exception, **kwargs)
     76 
     77     # Cleanup pools associated to dead threads

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    519                         _execute_task(task, data)  # Re-execute locally
    520                     else:
--> 521                         raise_exception(exc, tb)
    522                 res, worker_id = loads(res_info)
    523                 state['cache'][key] = res

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/compatibility.py in reraise(exc, tb)
     67         if exc.__traceback__ is not tb:
     68             raise exc.with_traceback(tb)
---> 69         raise exc
     70 
     71 else:

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    288     try:
    289         task, data = loads(task_info)
--> 290         result = _execute_task(task, data)
    291         id = get_id()
    292         result = dumps((result, id))

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/local.py in _execute_task(arg, cache, dsk)
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in getter_inline(a, b, asarray, lock)
    118     This inlining can be relevant to operations when running off of disk.
    119     """
--> 120     return getter(a, b, asarray=asarray, lock=lock)
    121 
    122 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in getter(a, b, asarray, lock)
     83         c = a[b]
     84         if asarray:
---> 85             c = np.asarray(c)
     86     finally:
     87         if lock:

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/numpy/core/numeric.py in asarray(a, dtype, order)
    499 
    500     """
--> 501     return array(a, dtype, copy=False, order=order)
    502 
    503 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/core.py in __array__(self, dtype, **kwargs)
   1207 
   1208     def __array__(self, dtype=None, **kwargs):
-> 1209         x = self.compute()
   1210         if dtype and x.dtype != dtype:
   1211             x = x.astype(dtype)

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(self, **kwargs)
    154         dask.base.compute
    155         """
--> 156         (result,) = compute(self, traverse=False, **kwargs)
    157         return result
    158 

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
    397                              collections=collections)
    398 
--> 399     dsk = collections_to_dsk(collections, optimize_graph, **kwargs)
    400     keys = [x.__dask_keys__() for x in collections]
    401     postcomputes = [x.__dask_postcompute__() for x in collections]

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in collections_to_dsk(collections, optimize_graph, **kwargs)
    192 
    193         dsk = merge(*(opt(dsk, keys, **kwargs)
--> 194                       for opt, (dsk, keys) in groups.items()))
    195     else:
    196         dsk, _ = _extract_graph_and_keys(collections)

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/base.py in <genexpr>(.0)
    192 
    193         dsk = merge(*(opt(dsk, keys, **kwargs)
--> 194                       for opt, (dsk, keys) in groups.items()))
    195     else:
    196         dsk, _ = _extract_graph_and_keys(collections)

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/optimization.py in optimize(dsk, keys, fuse_keys, fast_functions, inline_functions_fast_functions, rename_fused_keys, **kwargs)
     44     else:
     45         dsk4 = dsk3
---> 46     dsk5 = optimize_slices(dsk4)
     47 
     48     return dsk5

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/optimization.py in optimize_slices(dsk)
    132                     break
    133                 try:
--> 134                     c_index = fuse_slice(b_index, a_index)
    135                     # rely on fact that nested gets never decrease in
    136                     # strictness e.g. `(getter_nofancy, (getter, ...))` never

~/miniconda3/envs/cenv3/lib/python3.6/site-packages/dask/array/optimization.py in fuse_slice(a, b)
    280                 result.append(a[i])
    281                 continue
--> 282             while b[j] is None:  # insert any Nones on the rhs
    283                 result.append(None)
    284                 j += 1

IndexError: tuple index out of range
```

Python: 3.6.2
Dask: 0.18.1


Discussion:
rainwoodman: Sure. I am thinking of only doing the first fix (int cast) in this PR, and then use another PR to change all instance type checks to `Integral` from 'int'.
mrocklin: > Sure. I am thinking of only doing the first fix (int cast) in this PR, and then use another PR to change all instance type checks to Integral from 'int'.

No objection from me.  I would like to include this PR in a bugfix release that goes out on Friday if possible.  My guess is that we'll get things in easily by that time though.

Patch:
diff --git a/dask/array/slicing.py b/dask/array/slicing.py
index 964878798..4e170316f 100644
--- a/dask/array/slicing.py
+++ b/dask/array/slicing.py
@@ -391,7 +391,7 @@ def _slice_1d(dim_shape, lengths, index):
             ind = index - chunk_boundaries[i - 1]
         else:
             ind = index
-        return {i: ind}
+        return {int(i): int(ind)}
 
     assert isinstance(index, slice)
 


Test patch:
diff --git a/dask/array/tests/test_optimization.py b/dask/array/tests/test_optimization.py
index 5ba01d564..63bcb2686 100644
--- a/dask/array/tests/test_optimization.py
+++ b/dask/array/tests/test_optimization.py
@@ -273,3 +273,15 @@ def test_turn_off_fusion():
 
     assert dask.get(a, y.__dask_keys__()) == dask.get(b, y.__dask_keys__())
     assert len(a) < len(b)
+
+
+def test_gh3937():
+    # test for github issue #3937
+    x = da.from_array([1, 2, 3.], (2,))
+    x = da.concatenate((x, [x[-1]]))
+    y = x.rechunk((2,))
+    # This will produce Integral type indices that are not ints (np.int64), failing
+    # the optimizer
+    y = da.coarsen(np.sum, y, {0: 2})
+    # How to trigger the optimizer explicitly?
+    y.compute()