Repository: dask/dask

Issue:
Special case apply in pprint_task
When looking at profile plots, tasks that use apply become illegible

```
f(x, y, z=10) # want to specify this
(f, x, y, ??) # no way to specify kwargs
(apply, f, (x, y), {'z': 10})  # so we use apply!
(apply, f, (tuple, [x, y]), (dict, (list, [(list, ['x'])]), (list, [(list, [10])])])))  # but we also quote 
```

:(

We might consider special casing `apply` in the `pprint_task` function.  We might also consider a general `unquote` operation that would unpack all of the dict/list junk.  

cc @jcrist 



Discussion:
Unquote:

``` python
def unquote(expr):
    if istask(expr):
        if expr[0] in (tuple, list, set):
            return expr[0](map(unquote, expr[1]))
        elif expr[0] == dict and expr[1][0] == list:
            return dict(map(unquote, expr[1][1]))
    return expr
```

`pprint_task` will probably have to use something different, to properly handle printing keys and things. Not sure.

Perhaps `pprint_task` just includes recursive cases for `(list, ...)` and `(dict, ...)`.

For tasks with `apply`, would you want them to show `apply`? Or convert to normal call syntax, and show the dict as kwargs? I can see the merit in both options, just need a decision.

I suspect that most users are unfamiliar with `apply`.  I would opt to show them `f(x, y, z=10)` instead if possible.


Resolving patch:
diff --git a/dask/diagnostics/profile_visualize.py b/dask/diagnostics/profile_visualize.py
--- a/dask/diagnostics/profile_visualize.py
+++ b/dask/diagnostics/profile_visualize.py
@@ -11,6 +11,16 @@
 
 from ..dot import funcname
 from ..core import istask
+from ..compatibility import apply
+
+
+def unquote(expr):
+    if istask(expr):
+        if expr[0] in (tuple, list, set):
+            return expr[0](map(unquote, expr[1]))
+        elif expr[0] == dict and expr[1][0] == list:
+            return dict(map(unquote, expr[1][1]))
+    return expr
 
 
 def pprint_task(task, keys, label_size=60):
@@ -49,22 +59,40 @@ def pprint_task(task, keys, label_size=60):
     """
     if istask(task):
         func = task[0]
-        if hasattr(func, 'funcs'):
-            head = '('.join(funcname(f) for f in func.funcs)
-            tail = ')'*len(func.funcs)
-        else:
-            head = funcname(task[0])
+        if func is apply:
+            head = funcname(task[1])
             tail = ')'
-        if task[1:]:
-            label_size2 = int((label_size - len(head) - len(tail)) / len(task[1:]))
+            args = unquote(task[2]) if len(task) > 2 else ()
+            kwargs = unquote(task[3]) if len(task) > 3 else {}
+        else:
+            if hasattr(func, 'funcs'):
+                head = '('.join(funcname(f) for f in func.funcs)
+                tail = ')'*len(func.funcs)
+            else:
+                head = funcname(task[0])
+                tail = ')'
+            args = task[1:]
+            kwargs = {}
+        if args or kwargs:
+            label_size2 = int((label_size - len(head) - len(tail)) //
+                              (len(args) + len(kwargs)))
+            pprint = lambda t: pprint_task(t, keys, label_size2)
+        if args:
             if label_size2 > 5:
-                args = ', '.join(pprint_task(t, keys, label_size2)
-                                 for t in task[1:])
+                args = ', '.join(pprint(t) for t in args)
             else:
                 args = '...'
         else:
             args = ''
-        return '{0}({1}{2}'.format(head, args, tail)
+        if kwargs:
+            if label_size2 > 5:
+                kwargs = ', ' + ', '.join('{0}={1}'.format(k, pprint(v))
+                                          for k, v in sorted(kwargs.items()))
+            else:
+                kwargs = ', ...'
+        else:
+            kwargs = ''
+        return '{0}({1}{2}{3}'.format(head, args, kwargs, tail)
     elif isinstance(task, list):
         if not task:
             return '[]'
@@ -72,7 +100,7 @@ def pprint_task(task, keys, label_size=60):
             result = pprint_task(task[:3], keys, label_size)
             return result[:-1] + ', ...]'
         else:
-            label_size2 = int((label_size - 2 - 2*len(task)) / len(task))
+            label_size2 = int((label_size - 2 - 2*len(task)) // len(task))
             args = ', '.join(pprint_task(t, keys, label_size2) for t in task)
             return '[{0}]'.format(args)
     else:


Test patch:
diff --git a/dask/diagnostics/tests/test_profiler.py b/dask/diagnostics/tests/test_profiler.py
--- a/dask/diagnostics/tests/test_profiler.py
+++ b/dask/diagnostics/tests/test_profiler.py
@@ -5,12 +5,17 @@
 from dask.diagnostics import Profiler, ResourceProfiler
 from dask.threaded import get
 from dask.utils import ignoring, tmpfile
+from dask.compatibility import apply
 import pytest
 
 try:
     import bokeh
 except:
     bokeh = None
+try:
+    import psutil
+except:
+    psutil = None
 
 
 prof = Profiler()
@@ -69,6 +74,7 @@ def test_two_gets():
     assert len(prof.results) == n + m + n
 
 
+@pytest.mark.skipif("not psutil")
 def test_resource_profiler():
     with ResourceProfiler(dt=0.01) as rprof:
         out = get(dsk2, 'c')
@@ -86,6 +92,19 @@ def test_resource_profiler():
             get(dsk, 'e')
 
 
+@pytest.mark.skipif("not bokeh")
+def test_unquote():
+    from dask.diagnostics.profile_visualize import unquote
+    from dask.imperative import to_task_dasks
+    f = lambda x: to_task_dasks(x)[0]
+    t = {'a': 1, 'b': 2, 'c': 3}
+    assert unquote(f(t)) == t
+    t = {'a': [1, 2, 3], 'b': 2, 'c': 3}
+    assert unquote(f(t)) == t
+    t = [1, 2, 3]
+    assert unquote(f(t)) == t
+
+
 @pytest.mark.skipif("not bokeh")
 def test_pprint_task():
     from dask.diagnostics.profile_visualize import pprint_task
@@ -100,8 +119,17 @@ def test_pprint_task():
     assert pprint_task((sum, list(keys) * 100), keys) == 'sum([_, _, _, ...])'
     assert pprint_task((sum, [1, 2, (sum, ['a', 4]), 5, 6] * 100), keys) == \
             'sum([*, *, sum([_, *]), ...])'
-    assert pprint_task((sum, [1, 2, (sum, ['a', (sum, [1, 2, 3])]), 5, 6])
-                      , keys) == 'sum([*, *, sum([_, sum(...)]), ...])'
+    assert pprint_task((sum, [1, 2, (sum, ['a', (sum, [1, 2, 3])]), 5, 6]),
+                       keys) == 'sum([*, *, sum([_, sum(...)]), ...])'
+    # With kwargs
+    def foo(w, x, y=(), z=3):
+        return w + x + sum(y) + z
+    task = (apply, foo, (tuple, ['a', 'b']), (dict, (list,
+            [(list, ['y', (list, ['a', 'b'])]), (list, ['z', 'c'])])))
+    assert pprint_task(task, keys) == 'foo(_, _, y=[_, _], z=_)'
+    task = (apply, foo, (tuple, ['a', 'b']), (dict, (list,
+            [(list, ['y', (list, ['a', 1])]), (list, ['z', 1])])))
+    assert pprint_task(task, keys) == 'foo(_, _, y=[_, *], z=*)'
 
 
 @pytest.mark.skipif("not bokeh")
@@ -121,6 +149,7 @@ def test_profiler_plot():
 
 
 @pytest.mark.skipif("not bokeh")
+@pytest.mark.skipif("not psutil")
 def test_resource_profiler_plot():
     with ResourceProfiler(dt=0.01) as rprof:
         get(dsk2, 'c')
@@ -137,6 +166,7 @@ def test_resource_profiler_plot():
 
 
 @pytest.mark.skipif("not bokeh")
+@pytest.mark.skipif("not psutil")
 def test_plot_both():
     from dask.diagnostics.profile_visualize import visualize
     from bokeh.plotting import GridPlot