Repository: dask/distributed

Issue:
'Future' object has no attribute 'memory_usage'
```
import dask.dataframe as dd
import pandas as pd
from dask.distributed import Client

if __name__ == '__main__':
    client = Client()
    df = pd.DataFrame({'A': range(1, 11)})
    ddf = dd.from_pandas(df, npartitions=2)
    print(ddf.memory_usage().compute())

    pddf = ddf.persist()
    print(pddf.memory_usage().compute())
```
```
A         80
Index    164
dtype: int64
distributed.worker - WARNING -  Compute Failed
Function:  subgraph_callable
args:      ()
kwargs:    {}
Exception: AttributeError("'Future' object has no attribute 'memory_usage'",)

distributed.worker - WARNING -  Compute Failed
Function:  subgraph_callable
args:      ()
kwargs:    {}
Exception: AttributeError("'Future' object has no attribute 'memory_usage'",)

Traceback (most recent call last):
  File "mem.py", line 12, in <module>
    print(pddf.memory_usage().compute())
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/base.py", line 156, in compute
    (result,) = compute(self, traverse=False, **kwargs)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/base.py", line 397, in compute
    results = schedule(dsk, keys, **kwargs)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 2338, in get
    direct=direct)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 1662, in gather
    asynchronous=asynchronous)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 676, in sync
    return sync(self.loop, func, *args, **kwargs)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/utils.py", line 277, in sync
    six.reraise(*error[0])
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/six.py", line 693, in reraise
    raise value
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/utils.py", line 262, in f
    result[0] = yield future
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/tornado/gen.py", line 1133, in run
    value = future.result()
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/tornado/gen.py", line 1141, in run
    yielded = self.gen.throw(*exc_info)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 1503, in _gather
    traceback)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/six.py", line 692, in reraise
    raise value.with_traceback(tb)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/optimization.py", line 942, in __call__
    dict(zip(self.inkeys, args)))
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 132, in _get_recursive
    res = cache[x] = _get_recursive(dsk, dsk[x], cache)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in _get_recursive
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in <listcomp>
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in _get_recursive
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in <listcomp>
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 132, in _get_recursive
    res = cache[x] = _get_recursive(dsk, dsk[x], cache)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 137, in _get_recursive
    return func(*args2)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/dataframe/core.py", line 3684, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/utils.py", line 694, in __call__
    return getattr(obj, self.method)(*args, **kwargs)
AttributeError: 'Future' object has no attribute 'memory_usage'
distributed.process - WARNING - reaping stray process <ForkServerProcess(ForkServerProcess-1, started daemon)>
distributed.nanny - WARNING - Worker process 6135 was killed by unknown signal
```


Discussion:
It seems it's not limited to `memory_usage`; column (re)assignment on persisted dataframes may also fail:

```
import dask.dataframe as dd
import pandas as pd
from dask.distributed import Client

def square(df, *columns):
    for c in columns:
        df[c+c] = df[c]**2
    return df

if __name__ == '__main__':
    client = Client()
    df = pd.DataFrame({'A': range(1, 11), 'B': range(1, 11), })
    ddf = dd.from_pandas(df, npartitions=2)
    pddf = ddf.persist()

    print(ddf.compute())
    print(square(ddf, 'A').compute())
    print(square(ddf, 'A', 'B').compute())

    print(pddf.compute())
    # this works
    print(square(pddf, 'A').compute())
    # this doesn't
    print(square(pddf, 'A', 'B').compute())
```
```
    A   B
0   1   1
1   2   2
2   3   3
3   4   4
4   5   5
5   6   6
6   7   7
7   8   8
8   9   9
9  10  10
    A   B   AA
0   1   1    1
1   2   2    4
2   3   3    9
3   4   4   16
4   5   5   25
5   6   6   36
6   7   7   49
7   8   8   64
8   9   9   81
9  10  10  100
    A   B   AA   BB
0   1   1    1    1
1   2   2    4    4
2   3   3    9    9
3   4   4   16   16
4   5   5   25   25
5   6   6   36   36
6   7   7   49   49
7   8   8   64   64
8   9   9   81   81
9  10  10  100  100
    A   B
0   1   1
1   2   2
2   3   3
3   4   4
4   5   5
5   6   6
6   7   7
7   8   8
8   9   9
9  10  10
    A   B   AA
0   1   1    1
1   2   2    4
2   3   3    9
3   4   4   16
4   5   5   25
5   6   6   36
6   7   7   49
7   8   8   64
8   9   9   81
9  10  10  100
distributed.worker - WARNING -  Compute Failed
Function:  subgraph_callable
args:      ()
kwargs:    {}
Exception: TypeError("'Future' object is not subscriptable",)
distributed.worker - WARNING -  Compute Failed
Function:  subgraph_callable
args:      ()
kwargs:    {}
Exception: TypeError("'Future' object is not subscriptable",)


Traceback (most recent call last):
  File "mem.py", line 22, in <module>
    print(square(pddf, 'A', 'B').compute())
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/base.py", line 156, in compute
    (result,) = compute(self, traverse=False, **kwargs)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/base.py", line 397, in compute
    results = schedule(dsk, keys, **kwargs)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 2338, in get
    direct=direct)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 1662, in gather
    asynchronous=asynchronous)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 676, in sync
    return sync(self.loop, func, *args, **kwargs)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/utils.py", line 277, in sync
    six.reraise(*error[0])
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/six.py", line 693, in reraise
    raise value
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/utils.py", line 262, in f
    result[0] = yield future
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/tornado/gen.py", line 1133, in run
    value = future.result()
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/tornado/gen.py", line 1141, in run
    yielded = self.gen.throw(*exc_info)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/distributed/client.py", line 1503, in _gather
    traceback)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/six.py", line 692, in reraise
    raise value.with_traceback(tb)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/optimization.py", line 942, in __call__
    dict(zip(self.inkeys, args)))
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 132, in _get_recursive
    res = cache[x] = _get_recursive(dsk, dsk[x], cache)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in _get_recursive
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in <listcomp>
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in _get_recursive
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in <listcomp>
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 132, in _get_recursive
    res = cache[x] = _get_recursive(dsk, dsk[x], cache)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in _get_recursive
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in <listcomp>
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in _get_recursive
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in <listcomp>
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 132, in _get_recursive
    res = cache[x] = _get_recursive(dsk, dsk[x], cache)
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in _get_recursive
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in <listcomp>
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in _get_recursive
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in <listcomp>
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in _get_recursive
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 136, in <listcomp>
    args2 = [_get_recursive(dsk, k, cache) for k in args]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in _get_recursive
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 126, in <listcomp>
    return [_get_recursive(dsk, k, cache) for k in x]
  File "/home/gsk/projects/miniconda3/envs/myenv/lib/python3.6/site-packages/dask/core.py", line 137, in _get_recursive
    return func(*args2)
TypeError: 'Future' object is not subscriptable
distributed.process - WARNING - reaping stray process <ForkServerProcess(ForkServerProcess-1, started daemon)>
distributed.process - WARNING - reaping stray process <ForkServerProcess(ForkServerProcess-3, started daemon)>
distributed.nanny - WARNING - Worker process 7212 was killed by unknown signal
distributed.nanny - WARNING - Worker process 7208 was killed by unknown signal
``` 
I'm able to reproduce.  Thank you for the excellent bug report @gsakkis 

@jcrist @TomAugspurger any chance either of you have time to look into this?
I'm also getting a related error when calling `.dropna(how='any')` on a persisted dask dataframe:

`AttributeError: 'Future' object has no attribute 'dropna'`

This error happened when rerunning a dataframe preprocessing script that previously worked earlier today. I realized that between then and when I got the error, my dask version had been upgraded to `1.1.0`. When I rolled back to v`1.0.0` my script ran without error. @gsakkis what version of dask are you currently running? I wonder if this "fix" will work for you too. 

Sorry for not being able to produce more info, I'm currently under a tight deadline and needed to roll back to a stable version asap. I'll be following this thread and I'm happy to help investigate once my schedule frees up.  
No, it's probably different then, I'm getting it on v1.0.0:
```
dask==1.0.0
distributed==1.25.2
```
Taking a look now.
The breaking commit was https://github.com/dask/dask/pull/3979

So, as a temporary workaround, set the `fuse_subgraphs=False` config.

```python
    with dask.config.set(fuse_subgraphs=False):
        print(pddf.memory_usage().compute())
```
Yeah, it looks like `Future` objects are getting pulled into subgraph callables, which the optimization routines can't inspect (or at least that's my guess)

```
distributed/tests/test_collections.py::test_gh2460
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> PDB set_trace (IO-capturing turned off) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
> /Users/mrocklin/workspace/distributed/distributed/worker.py(2100)execute()
-> self.exceptions[key] = result['exception']
(Pdb) pp key
("('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', "
 '0)')
(Pdb) pp self.tasks
{"('from_pandas-f42bf07c4c4c8e11a5f8ed2e2dae12fd', 1)": (<function execute_task at 0x10887b950>,
                                                         (    A
5   6
6   7
7   8
8   9
9  10,),
                                                         {}),
 "('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)": (subgraph_callable,
                                                                                               (),
                                                                                               {})}
(Pdb) pp self.tasks["('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)"]
(subgraph_callable, (), {})
(Pdb) pp self.tasks["('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)"][0]
subgraph_callable
(Pdb) pp self.tasks["('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)"][0].dsk
{('memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0): (subgraph_callable,
                                                        <Future: status: pending, key: ('from_pandas-f42bf07c4c4c8e11a5f8ed2e2dae12fd', 0)>,
                                                        'from_pandas-f42bf07c4c4c8e11a5f8ed2e2dae12fd'),
 ('series-groupby-sum-chunk-f6d2e309a0f2e4745fa2238cfb751d81', 0, 0, 0): (<function apply at 0x107d33488>,
                                                                          <function _apply_chunk at 0x31b279ea0>,
                                                                          [('memory_usage-ac35f72721bff85cecd6bebd84ee216c',
                                                                            0),
                                                                           (subgraph_callable,
                                                                            ('memory_usage-ac35f72721bff85cecd6bebd84ee216c',
                                                                             0),
                                                                            'index',
                                                                            'memory_usage-ac35f72721bff85cecd6bebd84ee216c')],
                                                                          {'chunk': <methodcaller: sum>,
                                                                           'columns': None})}
```

Here is a test and a pdb statement to get to the situation above (I recommend the test regardless).

```diff
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index cb51d62c..877a4db2 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -173,3 +173,12 @@ def test_sparse_arrays(c, s, a, b):
     future = c.compute(s.sum(axis=0)[:10])

     yield future
+
+
+@gen_cluster(client=True)
+def test_gh2460(c, s, a, b):
+    df = pd.DataFrame({'A': range(1, 11)})
+    ddf = dd.from_pandas(df, npartitions=2)
+
+    ddf = ddf.persist()
+    yield c.compute(ddf.memory_usage())
diff --git a/distributed/worker.py b/distributed/worker.py
index 12b41ca1..6eb1051f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2096,6 +2096,7 @@ class Worker(ServerNode):
                     self.transition(key, 'rescheduled', report=False)
                     self.release_key(key, report=False)
                 else:
+                    import pdb; pdb.set_trace()
                     self.exceptions[key] = result['exception']
                     self.tracebacks[key] = result['traceback']
                     logger.warning(" Compute Failed\n"
```

cc @jcrist 
I spoke with Jim about this briefly. I was hoping to get to it last week
but ran out of time.

On Tue, Jan 29, 2019 at 6:35 PM Matthew Rocklin <notifications@github.com>
wrote:

> Yeah, it looks like Future objects are getting pulled into subgraph
> callables, which the optimization routines can't inspect (or at least
> that's my guess)
>
> distributed/tests/test_collections.py::test_gh2460
> >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> PDB set_trace (IO-capturing turned off) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
> > /Users/mrocklin/workspace/distributed/distributed/worker.py(2100)execute()
> -> self.exceptions[key] = result['exception']
> (Pdb) pp key
> ("('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', "
>  '0)')
> (Pdb) pp self.tasks
> {"('from_pandas-f42bf07c4c4c8e11a5f8ed2e2dae12fd', 1)": (<function execute_task at 0x10887b950>,
>                                                          (    A
> 5   6
> 6   7
> 7   8
> 8   9
> 9  10,),
>                                                          {}),
>  "('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)": (subgraph_callable,
>                                                                                                (),
>                                                                                                {})}
> (Pdb) pp self.tasks["('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)"]
> (subgraph_callable, (), {})
> (Pdb) pp self.tasks["('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)"][0]
> subgraph_callable
> (Pdb) pp self.tasks["('from_pandas-series-groupby-sum-chunk-memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0)"][0].dsk
> {('memory_usage-ac35f72721bff85cecd6bebd84ee216c', 0): (subgraph_callable,
>                                                         <Future: status: pending, key: ('from_pandas-f42bf07c4c4c8e11a5f8ed2e2dae12fd', 0)>,
>                                                         'from_pandas-f42bf07c4c4c8e11a5f8ed2e2dae12fd'),
>  ('series-groupby-sum-chunk-f6d2e309a0f2e4745fa2238cfb751d81', 0, 0, 0): (<function apply at 0x107d33488>,
>                                                                           <function _apply_chunk at 0x31b279ea0>,
>                                                                           [('memory_usage-ac35f72721bff85cecd6bebd84ee216c',
>                                                                             0),
>                                                                            (subgraph_callable,
>                                                                             ('memory_usage-ac35f72721bff85cecd6bebd84ee216c',
>                                                                              0),
>                                                                             'index',
>                                                                             'memory_usage-ac35f72721bff85cecd6bebd84ee216c')],
>                                                                           {'chunk': <methodcaller: sum>,
>                                                                            'columns': None})}
>
> Here is a test and a pdb statement to get to the situation above (I
> recommend the test regardless).
>
> diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
> index cb51d62c..877a4db2 100644--- a/distributed/tests/test_collections.py+++ b/distributed/tests/test_collections.py@@ -173,3 +173,12 @@ def test_sparse_arrays(c, s, a, b):
>      future = c.compute(s.sum(axis=0)[:10])
>
>      yield future+++@gen_cluster(client=True)+def test_gh2460(c, s, a, b):+    df = pd.DataFrame({'A': range(1, 11)})+    ddf = dd.from_pandas(df, npartitions=2)++    ddf = ddf.persist()+    yield c.compute(ddf.memory_usage())diff --git a/distributed/worker.py b/distributed/worker.py
> index 12b41ca1..6eb1051f 100644--- a/distributed/worker.py+++ b/distributed/worker.py@@ -2096,6 +2096,7 @@ class Worker(ServerNode):
>                      self.transition(key, 'rescheduled', report=False)
>                      self.release_key(key, report=False)
>                  else:+                    import pdb; pdb.set_trace()
>                      self.exceptions[key] = result['exception']
>                      self.tracebacks[key] = result['traceback']
>                      logger.warning(" Compute Failed\n"
>
> cc @jcrist <https://github.com/jcrist>
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/2460#issuecomment-458764415>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/ABQHIljVx8zdn27YPoxqXWUSb2ghYTULks5vIOjJgaJpZM4Z8vSK>
> .
>

@TomAugspurger, I'm going to take care of this.

Resolving patch:
diff --git a/distributed/client.py b/distributed/client.py
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -26,6 +26,7 @@
 import dask
 from dask.base import tokenize, normalize_token, collections_to_dsk
 from dask.core import flatten, get_dependencies
+from dask.optimization import SubgraphCallable
 from dask.compatibility import apply, unicode
 try:
     from cytoolz import first, groupby, merge, valmap, keymap
@@ -3964,11 +3965,13 @@ def futures_of(o, client=None):
         x = stack.pop()
         if type(x) in (tuple, set, list):
             stack.extend(x)
-        if type(x) is dict:
+        elif type(x) is dict:
             stack.extend(x.values())
-        if isinstance(x, Future):
+        elif type(x) is SubgraphCallable:
+            stack.extend(x.dsk.values())
+        elif isinstance(x, Future):
             futures.add(x)
-        if dask.is_dask_collection(x):
+        elif dask.is_dask_collection(x):
             stack.extend(x.__dask_graph__().values())
 
     if client is not None:
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -7,6 +7,7 @@
 from tornado import gen
 from tornado.gen import Return
 
+from dask.optimization import SubgraphCallable
 from toolz import merge, concat, groupby, drop
 
 from .core import rpc
@@ -177,11 +178,30 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
 
     typ = type(o)
 
+    if typ is tuple:
+        if not o:
+            return o
+        if type(o[0]) is SubgraphCallable:
+            sc = o[0]
+            futures = set()
+            dsk = {k: unpack_remotedata(v, byte_keys, futures)
+                   for k, v in sc.dsk.items()}
+            args = tuple(unpack_remotedata(i, byte_keys, futures) for i in o[1:])
+            if futures:
+                myset.update(futures)
+                futures = (tuple(tokey(f.key) for f in futures)
+                           if byte_keys else tuple(f.key for f in futures))
+                inkeys = sc.inkeys + futures
+                return (SubgraphCallable(dsk, sc.outkey, inkeys, sc.name),) + args + futures
+            else:
+                return o
+        else:
+            return tuple(unpack_remotedata(item, byte_keys, myset) for item in o)
     if typ in collection_types:
         if not o:
             return o
         outs = [unpack_remotedata(item, byte_keys, myset) for item in o]
-        return type(o)(outs)
+        return typ(outs)
     elif typ is dict:
         if o:
             values = [unpack_remotedata(v, byte_keys, myset) for v in o.values()]


Test patch:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -27,6 +27,7 @@
 
 import dask
 from dask import delayed
+from dask.optimization import SubgraphCallable
 import dask.bag as db
 from distributed import (Worker, Nanny, fire_and_forget, LocalCluster,
                          get_client, secede, get_worker, Executor, profile,
@@ -2493,7 +2494,7 @@ def test_wait_on_collections(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_futures_of(c, s, a, b):
+def test_futures_of_get(c, s, a, b):
     x, y, z = c.map(inc, [1, 2, 3])
 
     assert set(futures_of(0)) == set()
@@ -2505,6 +2506,11 @@ def test_futures_of(c, s, a, b):
     b = db.Bag({('b', i): f for i, f in enumerate([x, y, z])}, 'b', 3)
     assert set(futures_of(b)) == {x, y, z}
 
+    sg = SubgraphCallable({'x': x, 'y': y, 'z': z,
+                           'out': (add, (add, (add, x, y), z), 'in')},
+                          'out', ('in',))
+    assert set(futures_of(sg)) == {x, y, z}
+
 
 def test_futures_of_class():
     da = pytest.importorskip('dask.array')
@@ -5498,5 +5504,47 @@ def test_profile_bokeh(c, s, a, b):
         assert os.path.exists(fn)
 
 
+@gen_cluster(client=True)
+def test_get_mix_futures_and_SubgraphCallable(c, s, a, b):
+    future = c.submit(add, 1, 2)
+
+    subgraph = SubgraphCallable({'_2': (add, '_0', '_1'),
+                                 '_3': (add, future, '_2')},
+                                '_3', ('_0', '_1'))
+    dsk = {'a': 1,
+           'b': 2,
+           'c': (subgraph, 'a', 'b'),
+           'd': (subgraph, 'c', 'b')}
+
+    future2 = c.get(dsk, 'd', sync=False)
+    result = yield future2
+    assert result == 11
+
+    # Nested subgraphs
+    subgraph2 = SubgraphCallable({'_2': (subgraph, '_0', '_1'),
+                                  '_3': (subgraph, '_2', '_1'),
+                                  '_4': (add, '_3', future2)},
+                                 '_4', ('_0', '_1'))
+
+    dsk2 = {'e': 1, 'f': 2, 'g': (subgraph2, 'e', 'f')}
+
+    result = yield c.get(dsk2, 'g', sync=False)
+    assert result == 22
+
+
+@gen_cluster(client=True)
+def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
+    dd = pytest.importorskip('dask.dataframe')
+    import pandas as pd
+    df = pd.DataFrame({'x': range(1, 11)})
+    ddf = dd.from_pandas(df, npartitions=2).persist()
+    ddf = ddf.map_partitions(lambda x: x)
+    ddf['x'] = ddf['x'].astype('f8')
+    ddf = ddf.map_partitions(lambda x: x)
+    ddf['x'] = ddf['x'].astype('f8')
+    result = yield c.compute(ddf)
+    assert result.equals(df.astype('f8'))
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401