Repository: dask/dask

Issue:
Cannot compute map_overlap results with different overlaps from the same data together
Taken from issue ( https://github.com/dask/dask/issues/3633 )

So either there is a bug in my syntax below or there is a bug in `map_overlap`. Would be good to know either way.

<details>
<summary>Example:</summary>

```python
In [1]: import numpy as np

In [2]: import dask.array as da

In [3]: np.set_printoptions(linewidth=157)

In [4]: da.random.seed(0)

In [5]: a = da.random.random((9, 9), chunks=(3, 3))

In [6]: r = [
   ...:     a.map_overlap(
   ...:         lambda a: a + 1,
   ...:         dtype=a.dtype,
   ...:         depth={j: int(i == j) for j in range(a.ndim)},
   ...:         boundary="none"
   ...:     )
   ...:     for i in range(a.ndim)
   ...: ]

In [7]: r[0].compute()
Out[7]: 
array([[1.26183679, 1.66733012, 1.4331588 , 1.18722244, 1.3598777 , 1.62570796, 1.27312269, 1.05253473, 1.99576146],
       [1.88395965, 1.83022378, 1.47527108, 1.56056966, 1.45481825, 1.13954979, 1.39160789, 1.5152978 , 1.63736621],
       [1.94830662, 1.74161186, 1.31698935, 1.14048273, 1.11438004, 1.93272056, 1.32434274, 1.85906248, 1.68778608],
       [1.07752296, 1.09414421, 1.29101671, 1.67987996, 1.46675901, 1.02066416, 1.54502084, 1.43478848, 1.15347997],
       [1.97203482, 1.75125324, 1.1479907 , 1.49482728, 1.29174358, 1.8515408 , 1.70393362, 1.03947838, 1.33772461],
       [1.81670432, 1.65892727, 1.87848966, 1.89743835, 1.101832  , 1.05170626, 1.22893537, 1.83890211, 1.87083658],
       [1.32117875, 1.6571231 , 1.33603726, 1.91005159, 1.45899303, 1.26034124, 1.9476105 , 1.7051694 , 1.82094183],
       [1.4926356 , 1.44990117, 1.99980478, 1.18954519, 1.1126101 , 1.0731892 , 1.68447853, 1.78675853, 1.92631166],
       [1.3178578 , 1.8222106 , 1.17781999, 1.79586701, 1.13951092, 1.40103317, 1.12650002, 1.57615028, 1.65180453]])

In [8]: r[1].compute()
Out[8]: 
array([[1.26183679, 1.66733012, 1.4331588 , 1.18722244, 1.3598777 , 1.62570796, 1.27312269, 1.05253473, 1.99576146],
       [1.88395965, 1.83022378, 1.47527108, 1.56056966, 1.45481825, 1.13954979, 1.39160789, 1.5152978 , 1.63736621],
       [1.94830662, 1.74161186, 1.31698935, 1.14048273, 1.11438004, 1.93272056, 1.32434274, 1.85906248, 1.68778608],
       [1.07752296, 1.09414421, 1.29101671, 1.67987996, 1.46675901, 1.02066416, 1.54502084, 1.43478848, 1.15347997],
       [1.97203482, 1.75125324, 1.1479907 , 1.49482728, 1.29174358, 1.8515408 , 1.70393362, 1.03947838, 1.33772461],
       [1.81670432, 1.65892727, 1.87848966, 1.89743835, 1.101832  , 1.05170626, 1.22893537, 1.83890211, 1.87083658],
       [1.32117875, 1.6571231 , 1.33603726, 1.91005159, 1.45899303, 1.26034124, 1.9476105 , 1.7051694 , 1.82094183],
       [1.4926356 , 1.44990117, 1.99980478, 1.18954519, 1.1126101 , 1.0731892 , 1.68447853, 1.78675853, 1.92631166],
       [1.3178578 , 1.8222106 , 1.17781999, 1.79586701, 1.13951092, 1.40103317, 1.12650002, 1.57615028, 1.65180453]])

In [9]: da.compute(*r)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-9-91d1152d04ee> in <module>()
----> 1 da.compute(*r)

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
    398     keys = [x.__dask_keys__() for x in collections]
    399     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 400     results = schedule(dsk, keys, **kwargs)
    401     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    402 

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
     73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
     74                         cache=cache, get_id=_thread_get_id,
---> 75                         pack_exception=pack_exception, **kwargs)
     76 
     77     # Cleanup pools associated to dead threads

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    519                         _execute_task(task, data)  # Re-execute locally
    520                     else:
--> 521                         raise_exception(exc, tb)
    522                 res, worker_id = loads(res_info)
    523                 state['cache'][key] = res

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/compatibility.py in reraise(exc, tb)
     66         if exc.__traceback__ is not tb:
     67             raise exc.with_traceback(tb)
---> 68         raise exc
     69 
     70 else:

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    288     try:
    289         task, data = loads(task_info)
--> 290         result = _execute_task(task, data)
    291         id = get_id()
    292         result = dumps((result, id))

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/local.py in _execute_task(arg, cache, dsk)
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/local.py in <listcomp>(.0)
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/local.py in _execute_task(arg, cache, dsk)
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

/zopt/conda3/envs/test/lib/python3.6/site-packages/dask/array/core.py in concatenate3(arrays)
   3710             while arr.ndim < ndim:
   3711                 arr = arr[None, ...]
-> 3712         result[idx] = arr
   3713 
   3714     return result

ValueError: could not broadcast input array from shape (3,4) into shape (3,3)
```

With Dask 0.17.5, the computation completes, but the first array has the wrong shape.

```python
In [9]: da.compute(*r)
Out[9]: 
(array([[1.26183679, 1.66733012, 1.4331588 , 1.4331588 , 1.18722244, 1.3598777 , 1.62570796, 1.27312269, 1.05253473],
        [1.88395965, 1.83022378, 1.47527108, 1.47527108, 1.56056966, 1.45481825, 1.13954979, 1.39160789, 1.5152978 ],
        [1.97203482, 1.75125324, 1.1479907 , 1.1479907 , 1.49482728, 1.29174358, 1.8515408 , 1.70393362, 1.03947838],
        [1.4926356 , 1.44990117, 1.99980478, 1.99980478, 1.18954519, 1.1126101 , 1.0731892 , 1.68447853, 1.78675853],
        [1.3178578 , 1.8222106 , 1.17781999, 1.17781999, 1.79586701, 1.13951092, 1.40103317, 1.12650002, 1.57615028]]),
 array([[1.26183679, 1.66733012, 1.4331588 , 1.18722244, 1.3598777 , 1.62570796, 1.27312269, 1.05253473, 1.99576146],
        [1.88395965, 1.83022378, 1.47527108, 1.56056966, 1.45481825, 1.13954979, 1.39160789, 1.5152978 , 1.63736621],
        [1.94830662, 1.74161186, 1.31698935, 1.14048273, 1.11438004, 1.93272056, 1.32434274, 1.85906248, 1.68778608],
        [1.07752296, 1.09414421, 1.29101671, 1.67987996, 1.46675901, 1.02066416, 1.54502084, 1.43478848, 1.15347997],
        [1.97203482, 1.75125324, 1.1479907 , 1.49482728, 1.29174358, 1.8515408 , 1.70393362, 1.03947838, 1.33772461],
        [1.81670432, 1.65892727, 1.87848966, 1.89743835, 1.101832  , 1.05170626, 1.22893537, 1.83890211, 1.87083658],
        [1.32117875, 1.6571231 , 1.33603726, 1.91005159, 1.45899303, 1.26034124, 1.9476105 , 1.7051694 , 1.82094183],
        [1.4926356 , 1.44990117, 1.99980478, 1.18954519, 1.1126101 , 1.0731892 , 1.68447853, 1.78675853, 1.92631166],
        [1.3178578 , 1.8222106 , 1.17781999, 1.79586701, 1.13951092, 1.40103317, 1.12650002, 1.57615028, 1.65180453]]))
```

</details>

<br>

<details>
<summary>Environment:</summary>

```yaml
name: test
channels:
  - conda-forge
  - defaults
dependencies:
  - appnope=0.1.0=py36_0
  - asciitree=0.3.3=py36_1
  - backcall=0.1.0=py_0
  - blas=1.1=openblas
  - bokeh=0.12.16=py36_0
  - ca-certificates=2018.4.16=0
  - certifi=2018.4.16=py36_0
  - click=6.7=py_1
  - cloudpickle=0.5.3=py_0
  - cytoolz=0.9.0.1=py36_0
  - dask=0.18.0=py_0
  - dask-core=0.18.0=py_0
  - decorator=4.3.0=py_0
  - distributed=1.22.0=py36_0
  - fasteners=0.14.1=py36_2
  - heapdict=1.0.0=py36_0
  - ipython=6.4.0=py36_0
  - ipython_genutils=0.2.0=py36_0
  - jedi=0.12.0=py36_0
  - jinja2=2.10=py36_0
  - libgfortran=3.0.0=0
  - locket=0.2.0=py36_1
  - markupsafe=1.0=py36_0
  - monotonic=1.5=py_0
  - msgpack-python=0.5.6=py36h2d50403_2
  - ncurses=5.9=10
  - numcodecs=0.5.5=py36_0
  - numpy=1.14.5=py36_blas_openblash24bf2e0_200
  - openblas=0.2.20=8
  - openssl=1.0.2o=0
  - packaging=17.1=py_0
  - pandas=0.23.1=py36_0
  - parso=0.2.1=py_0
  - partd=0.3.8=py36_0
  - pexpect=4.6.0=py36_0
  - pickleshare=0.7.4=py36_0
  - prompt_toolkit=1.0.15=py36_0
  - psutil=5.4.6=py36_0
  - ptyprocess=0.5.2=py36_0
  - pygments=2.2.0=py36_0
  - pyparsing=2.2.0=py36_0
  - python=3.6.5=1
  - python-dateutil=2.7.3=py_0
  - pytz=2018.4=py_0
  - pyyaml=3.12=py36_1
  - readline=7.0=0
  - setuptools=39.2.0=py36_0
  - simplegeneric=0.8.1=py36_0
  - six=1.11.0=py36_1
  - sortedcontainers=2.0.3=py36_0
  - sqlite=3.20.1=2
  - tblib=1.3.2=py36_0
  - tk=8.6.7=0
  - toolz=0.9.0=py_0
  - tornado=5.0.2=py36_0
  - traitlets=4.3.2=py36_0
  - wcwidth=0.1.7=py36_0
  - xz=5.2.3=0
  - yaml=0.1.7=0
  - zarr=2.2.0=py_1
  - zict=0.1.3=py_0
  - zlib=1.2.11=h470a237_3
```

</details>


Discussion:

Resolving patch:
diff --git a/dask/array/ghost.py b/dask/array/ghost.py
--- a/dask/array/ghost.py
+++ b/dask/array/ghost.py
@@ -49,17 +49,28 @@ def fractional_slice(task, axes):
         return (getitem, rounded, index)
 
 
-def expand_key(k, dims):
+def expand_key(k, dims, name=None):
     """ Get all neighboring keys around center
 
-    >>> expand_key(('x', 2, 3), dims=[5, 5])  # doctest: +NORMALIZE_WHITESPACE
-    [[('x', 1.1, 2.1), ('x', 1.1, 3), ('x', 1.1, 3.9)],
-     [('x',   2, 2.1), ('x',   2, 3), ('x',   2, 3.9)],
-     [('x', 2.9, 2.1), ('x', 2.9, 3), ('x', 2.9, 3.9)]]
+    Parameters
+    ----------
+    k: tuple
+        They key around which to generate new keys
+    dims: Sequence[int]
+        The number of chunks in each dimension
+    name: Option[str]
+        The name to include in the output keys, or none to include no name
 
-    >>> expand_key(('x', 0, 4), dims=[5, 5])  # doctest: +NORMALIZE_WHITESPACE
-    [[('x',   0, 3.1), ('x',   0,   4)],
-     [('x', 0.9, 3.1), ('x', 0.9,   4)]]
+    Examples
+    --------
+    >>> expand_key(('x', 2, 3), dims=[5, 5], name='y')  # doctest: +NORMALIZE_WHITESPACE
+    [[('y', 1.1, 2.1), ('y', 1.1, 3), ('y', 1.1, 3.9)],
+     [('y',   2, 2.1), ('y',   2, 3), ('y',   2, 3.9)],
+     [('y', 2.9, 2.1), ('y', 2.9, 3), ('y', 2.9, 3.9)]]
+
+    >>> expand_key(('x', 0, 4), dims=[5, 5], name='y')  # doctest: +NORMALIZE_WHITESPACE
+    [[('y',   0, 3.1), ('y',   0,   4)],
+     [('y', 0.9, 3.1), ('y', 0.9,   4)]]
     """
     def inds(i, ind):
         rv = []
@@ -79,8 +90,10 @@ def inds(i, ind):
             num += 1
         shape.append(num)
 
-    seq = list(product([k[0]], *[inds(i, ind)
-                                 for i, ind in enumerate(k[1:])]))
+    args = [inds(i, ind) for i, ind in enumerate(k[1:])]
+    if name is not None:
+        args = [[name]] + args
+    seq = list(product(*args))
     return reshapelist(shape, seq)
 
 
@@ -100,21 +113,23 @@ def ghost_internal(x, axes):
     """
     dims = list(map(len, x.chunks))
     expand_key2 = partial(expand_key, dims=dims)
+
+    # Make keys for each of the surrounding sub-arrays
     interior_keys = pipe(x.__dask_keys__(), flatten, map(expand_key2),
                          map(flatten), concat, list)
 
-    token = tokenize(x, axes)
-    name = 'ghost-' + token
+    name = 'ghost-' + tokenize(x, axes)
+    getitem_name = 'getitem-' + tokenize(x, axes)
     interior_slices = {}
     ghost_blocks = {}
     for k in interior_keys:
-        frac_slice = fractional_slice(k, axes)
-        if k != frac_slice:
-            interior_slices[k] = frac_slice
-
+        frac_slice = fractional_slice((x.name,) + k, axes)
+        if (x.name,) + k != frac_slice:
+            interior_slices[(getitem_name,) + k] = frac_slice
         else:
-            ghost_blocks[(name,) + k[1:]] = (concatenate3,
-                                             (concrete, expand_key2(k)))
+            interior_slices[(getitem_name,) + k] = (x.name,) + k
+            ghost_blocks[(name,) + k] = (concatenate3,
+                                         (concrete, expand_key2((None,) + k, name=getitem_name)))
 
     chunks = []
     for i, bds in enumerate(x.chunks):


Test patch:
diff --git a/dask/array/tests/test_ghost.py b/dask/array/tests/test_ghost.py
--- a/dask/array/tests/test_ghost.py
+++ b/dask/array/tests/test_ghost.py
@@ -365,3 +365,21 @@ def test_ghost_small():
 
     y = x.map_overlap(lambda x: x, depth=1, boundary='none')
     assert len(y.dask) < 100
+
+
+def test_no_shared_keys_with_different_depths():
+    da.random.seed(0)
+    a = da.random.random((9, 9), chunks=(3, 3))
+
+    def check(x):
+        assert x.shape == (3, 3)
+        return x
+
+    r = [a.map_overlap(lambda a: a + 1,
+                       dtype=a.dtype,
+                       depth={j: int(i == j) for j in range(a.ndim)},
+                       boundary="none").map_blocks(check, dtype=a.dtype)
+         for i in range(a.ndim)]
+
+    assert set(r[0].dask) & set(r[1].dask) == set(a.dask)
+    da.compute(*r, scheduler='single-threaded')