Repository: dask/dask

Issue:
Scheduler doesn't compute lists of tasks
Lists of tasks with no external dependencies aren't computed:

``` python
In [1]: from dask import get

In [2]: def inc(x):
   ...:     return x + 1
   ...:

# This doesn't work
In [3]: dsk = {'a': [1, (inc, 1)]}

In [4]: get(dsk, 'a')
Out[4]: [1, (<function __main__.inc>, 1)]

# But this does
In [5]: dsk = {'a': ['b', (inc, 1)], 'b': 1}

In [6]: get(dsk, 'a')
Out[6]: [1, 2]

# And so does this
In [7]: dsk = {'a': [1, (inc, 'b')], 'b': 1}

In [8]: get(dsk, 'a')
Out[8]: [1, 2]
```

Discovered while investigating http://stackoverflow.com/questions/37535160/using-dask-multi-threaded-module.



Discussion:
Related: https://github.com/dask/dask/issues/421. I still think this is inconsistent here, not sure what the best solution is. Past me thought it was don't compute things that aren't tasks (meaning no top-level lists should be computed).

Because of this thought, I've pushed a fix for the stackoverflow issue here https://github.com/dask/dask/pull/1194, which fixes it so that those graphs don't result.

This issue should stay open for discussion though (which is really a continuation of https://github.com/dask/dask/issues/421). My personal preference is that either all or none of the above work. None seems safer, as the list could be extremely large and nested.

My assumption was that `{'a': [1, (inc, 1)]}` _would_ trigger a computation.  Performance issues aside this seems least surprising to me.  

Have we run into cases where long list traversal burdens pragmatic problems?

Not that I know of, but I could probably construct some artificial case where it would cause problems. It's also a bit ambiguous what a list of strings represents:

``` python
{'a': ['b', 'c', 'd', 'e'], ...}
```

Are the strings keys or data? You'd have to check each one to see if it's a key in the graph. This may not be bad, it's just something that may come up.

Having list literals be tasks regardless of whether they're a direct argument or not would definitely simplify things. If this is a but, it's been broken this way for a while (see #421).

Per the spec `'b'` is considered to be a key if its in the dict or a literal if it isn't.  Not the most explicit, but simple and fast-ish to check.


Resolving patch:
diff --git a/dask/async.py b/dask/async.py
--- a/dask/async.py
+++ b/dask/async.py
@@ -119,7 +119,7 @@
 import traceback
 
 from .core import (istask, flatten, reverse_dict, get_dependencies, ishashable,
-                   _deps)
+                   has_tasks)
 from .context import _globals
 from .order import order
 from .callbacks import unpack_callbacks
@@ -168,7 +168,7 @@ def start_state_from_dask(dsk, cache=None, sortkey=None):
         cache = dict()
     data_keys = set()
     for k, v in dsk.items():
-        if not (istask(v) or _deps(dsk, v)):
+        if not has_tasks(dsk, v):
             cache[k] = v
             data_keys.add(k)
 
diff --git a/dask/core.py b/dask/core.py
--- a/dask/core.py
+++ b/dask/core.py
@@ -42,6 +42,28 @@ def istask(x):
     return type(x) is tuple and x and callable(x[0])
 
 
+def has_tasks(dsk, x):
+    """Whether ``x`` has anything to compute.
+
+    Returns True if:
+    - ``x`` is a task
+    - ``x`` is a key in ``dsk``
+    - ``x`` is a list that contains any tasks or keys
+    """
+    if istask(x):
+        return True
+    try:
+        if x in dsk:
+            return True
+    except:
+        pass
+    if isinstance(x, list):
+        for i in x:
+            if has_tasks(dsk, i):
+                return True
+    return False
+
+
 def preorder_traversal(task):
     """A generator to preorder-traverse a task."""
 
@@ -57,9 +79,9 @@ def preorder_traversal(task):
             yield item
 
 
-def _get_task(d, task, maxdepth=1000):
-    # non-recursive.  DAG property is checked upon reaching maxdepth.
-    _iter = lambda *args: iter(args)
+def _get_nonrecursive(d, x, maxdepth=1000):
+    # Non-recursive. DAG property is checked upon reaching maxdepth.
+    _list = lambda *args: list(args)
 
     # We construct a nested heirarchy of tuples to mimic the execution stack
     # of frames that Python would maintain for a recursive implementation.
@@ -70,7 +92,7 @@ def _get_task(d, task, maxdepth=1000):
     #       Arguments are stored in reverse order, and elements are popped
     #       as they are evaluated.
     #    3) The calculated results of the arguments from (2).
-    stack = [(task[0], list(task[:0:-1]), [])]
+    stack = [(lambda x: x, [x], [])]
     while True:
         func, args, results = stack[-1]
         if not args:
@@ -81,7 +103,7 @@ def _get_task(d, task, maxdepth=1000):
             stack[-1][2].append(val)
             continue
         elif maxdepth and len(stack) > maxdepth:
-            cycle = getcycle(d, list(task[1:]))
+            cycle = getcycle(d, x)
             if cycle:
                 cycle = '->'.join(cycle)
                 raise RuntimeError('Cycle detected in Dask: %s' % cycle)
@@ -89,22 +111,32 @@ def _get_task(d, task, maxdepth=1000):
 
         key = args.pop()
         if isinstance(key, list):
-            # v = (get(d, k, concrete=False) for k in key)  # recursive
-            # Fake being lazy
-            stack.append((_iter, key[::-1], []))
+            stack.append((_list, list(key[::-1]), []))
             continue
         elif ishashable(key) and key in d:
-            v = d[key]
-        else:
-            v = key
-
-        if istask(v):
-            stack.append((v[0], list(v[:0:-1]), []))
+            args.append(d[key])
+            continue
+        elif istask(key):
+            stack.append((key[0], list(key[:0:-1]), []))
         else:
-            results.append(v)
+            results.append(key)
+
+
+def _get_recursive(d, x):
+    # recursive, no cycle detection
+    if isinstance(x, list):
+        return [_get_recursive(d, k) for k in x]
+    elif ishashable(x) and x in d:
+        return _get_recursive(d, d[x])
+    elif istask(x):
+        func, args = x[0], x[1:]
+        args2 = [_get_recursive(d, k) for k in args]
+        return func(*args2)
+    else:
+        return x
 
 
-def get(d, key, get=None, **kwargs):
+def get(d, x, recursive=False):
     """ Get value from Dask
 
     Examples
@@ -117,39 +149,13 @@ def get(d, key, get=None, **kwargs):
     1
     >>> get(d, 'y')
     2
-
-    See Also
-    --------
-    set
     """
-    get = get or _get
-    if isinstance(key, list):
-        v = tuple(get(d, k, get=get) for k in key)
-    elif istask(key):
-        v = key
-    elif ishashable(key):
-        v = d[key]
-    else:
-        message = '%s is neither a task or a dask key'
-        raise KeyError(message % key)
-
-    if istask(v):
-        if get is _get:
-            # use non-recursive method by default
-            return _get_task(d, v)
-        func, args = v[0], v[1:]
-
-        args2 = []
-        for arg in args:
-            if not istask(arg) and arg not in d:
-                args2.append(arg)
-            else:
-                args2.append(get(d, arg, get=get))
-        return func(*args2)
-    else:
-        return v
-
-_get = get
+    _get = _get_recursive if recursive else _get_nonrecursive
+    if isinstance(x, list):
+        return tuple(get(d, k) for k in x)
+    elif x in d:
+        return _get(d, x)
+    raise KeyError("{0} is not a key in the graph".format(x))
 
 
 def _deps(dsk, arg):


Test patch:
diff --git a/dask/tests/test_async.py b/dask/tests/test_async.py
--- a/dask/tests/test_async.py
+++ b/dask/tests/test_async.py
@@ -7,7 +7,7 @@
 import pytest
 
 from dask.async import *
-from dask.utils_test import GetFunctionTestCase
+from dask.utils_test import GetFunctionTestMixin
 
 
 fib_dask = {'f0': 0, 'f1': 1, 'f2': 1, 'f3': 2, 'f4': 3, 'f5': 5, 'f6': 8}
@@ -54,6 +54,18 @@ def test_start_state_with_independent_but_runnable_tasks():
     assert start_state_from_dask({'x': (inc, 1)})['ready'] == ['x']
 
 
+def test_start_state_with_tasks_no_deps():
+    dsk = {'a': [1, (inc, 2)],
+           'b': [1, 2, 3, 4],
+           'c': (inc, 3)}
+    state = start_state_from_dask(dsk)
+    assert list(state['cache'].keys()) == ['b']
+    assert 'a' in state['ready'] and 'c' in state['ready']
+    deps = dict((k, set()) for k in 'abc')
+    assert state['dependencies'] == deps
+    assert state['dependents'] == deps
+
+
 def test_finish_task():
     dsk = {'x': 1, 'y': 2, 'z': (inc, 'x'), 'w': (add, 'z', 'y')}
     sortkey = order(dsk).get
@@ -85,7 +97,8 @@ def test_finish_task():
           'waiting_data': {'y': set(['w']),
                            'z': set(['w'])}}
 
-class TestGetAsync(GetFunctionTestCase):
+
+class TestGetAsync(GetFunctionTestMixin):
     get = staticmethod(get_sync)
 
     def test_get_sync_num_workers(self):
diff --git a/dask/tests/test_core.py b/dask/tests/test_core.py
--- a/dask/tests/test_core.py
+++ b/dask/tests/test_core.py
@@ -1,10 +1,10 @@
 from collections import namedtuple
 
 from dask.utils import raises
-from dask.utils_test import GetFunctionTestCase
+from dask.utils_test import GetFunctionTestMixin
 from dask import core
 from dask.core import (istask, get_dependencies, flatten, subs,
-                       preorder_traversal, quote, _deps)
+                       preorder_traversal, quote, _deps, has_tasks)
 
 
 def contains(a, b):
@@ -17,9 +17,11 @@ def contains(a, b):
     """
     return all(a.get(k) == v for k, v in b.items())
 
+
 def inc(x):
     return x + 1
 
+
 def add(x, y):
     return x + y
 
@@ -32,6 +34,21 @@ def test_istask():
     assert not istask(f(sum, 2))
 
 
+def test_has_tasks():
+    dsk = {'a': [1, 2, 3],
+           'b': 'a',
+           'c': [1, (inc, 1)],
+           'd': [(sum, 'a')],
+           'e': ['a', 'b'],
+           'f': [['a', 'b'], 2, 3]}
+    assert not has_tasks(dsk, dsk['a'])
+    assert has_tasks(dsk, dsk['b'])
+    assert has_tasks(dsk, dsk['c'])
+    assert has_tasks(dsk, dsk['d'])
+    assert has_tasks(dsk, dsk['e'])
+    assert has_tasks(dsk, dsk['f'])
+
+
 def test_preorder_traversal():
     t = (add, 1, 2)
     assert list(preorder_traversal(t)) == [add, 1, 2]
@@ -41,55 +58,32 @@ def test_preorder_traversal():
     assert list(preorder_traversal(t)) == [add, sum, list, 1, 2, 3]
 
 
-class TestGet(GetFunctionTestCase):
+class TestGet(GetFunctionTestMixin):
     get = staticmethod(core.get)
 
 
-def test_GetFunctionTestCase_class():
-    class CustomTestGetFail(GetFunctionTestCase):
+class TestRecursiveGet(GetFunctionTestMixin):
+    get = staticmethod(lambda d, k: core.get(d, k, recursive=True))
+
+    def test_get_stack_limit(self):
+        # will blow stack in recursive mode
+        pass
+
+
+def test_GetFunctionTestMixin_class():
+    class TestCustomGetFail(GetFunctionTestMixin):
         get = staticmethod(lambda x, y: 1)
 
-    custom_testget = CustomTestGetFail()
+    custom_testget = TestCustomGetFail()
     raises(AssertionError, custom_testget.test_get)
 
-    class CustomTestGetPass(GetFunctionTestCase):
+    class TestCustomGetPass(GetFunctionTestMixin):
         get = staticmethod(core.get)
 
-    custom_testget = CustomTestGetPass()
+    custom_testget = TestCustomGetPass()
     custom_testget.test_get()
 
 
-def test_memoized_get():
-    d = {':x': 1,
-         ':y': (inc, ':x'),
-         ':z': (add, ':x', ':y')}
-    try:
-        import toolz
-    except ImportError:
-        return
-    cache = dict()
-
-    getm = toolz.memoize(core.get, cache=cache,
-                         key=lambda args, kwargs: args[1:])
-
-    result = getm(d, ':z', get=getm)
-    assert result == 3
-
-    assert contains(cache, {(':x',): 1,
-                            (':y',): 2,
-                            (':z',): 3})
-
-
-def test_get_laziness():
-    def isconcrete(arg):
-        return isinstance(arg, list)
-
-    d = {'x': 1, 'y': 2, 'z': (isconcrete, ['x', 'y'])}
-
-    assert core.get(d, ['x', 'y']) == (1, 2)
-    assert core.get(d, 'z') == False
-
-
 def test_get_dependencies_nested():
     dsk = {'x': 1, 'y': 2,
            'z': (add, (inc, [['x']]), 'y')}
diff --git a/dask/utils_test.py b/dask/utils_test.py
--- a/dask/utils_test.py
+++ b/dask/utils_test.py
@@ -1,25 +1,27 @@
 from __future__ import absolute_import, division, print_function
 
 from .core import get
-import unittest
+
 
 def inc(x):
     return x + 1
 
+
 def add(x, y):
     return x + y
 
-class GetFunctionTestCase(unittest.TestCase):
+
+class GetFunctionTestMixin(object):
     """
     The GetFunctionTestCase class can be imported and used to test foreign
     implementations of the `get` function specification. It aims to enforce all
-    known expecctations of `get` functions.
+    known expectations of `get` functions.
 
     To use the class, inherit from it and override the `get` function. For
     example:
 
-    > from dask.utils_test import GetFunctionTestCase
-    > class CustomGetTestCase(GetFunctionTestCase):
+    > from dask.utils_test import GetFunctionTestMixin
+    > class TestCustomGet(GetFunctionTestMixin):
          get = staticmethod(myget)
 
     Note that the foreign `myget` function has to be explicitly decorated as a
@@ -27,10 +29,6 @@ class GetFunctionTestCase(unittest.TestCase):
     """
     get = staticmethod(get)
 
-    def runTest(self):
-        """This method is needed for compatibility with PY2 unittest.TestCase"""
-        pass
-
     def test_get(self):
         d = {':x': 1,
              ':y': (inc, ':x'),
@@ -51,7 +49,7 @@ def test_badkey(self):
         else:
             msg = 'Expected `{}` with badkey to raise KeyError.\n'
             msg += "Obtained '{}' instead.".format(result)
-            self.assertTrue(False, msg=msg.format(self.get.__name__))
+            assert False, msg.format(self.get.__name__)
 
     def test_nested_badkey(self):
         d = {'x': 1, 'y': 2, 'z': (sum, ['x', 'y'])}
@@ -63,7 +61,7 @@ def test_nested_badkey(self):
         else:
             msg = 'Expected `{}` with badkey to raise KeyError.\n'
             msg += "Obtained '{}' instead.".format(result)
-            self.assertTrue(False, msg=msg.format(self.get.__name__))
+            assert False, msg.format(self.get.__name__)
 
     def test_data_not_in_dict_is_ok(self):
         d = {'x': 1, 'y': (add, 'x', 10)}
@@ -75,6 +73,20 @@ def test_get_with_list(self):
         assert self.get(d, ['x', 'y']) == (1, 2)
         assert self.get(d, 'z') == 3
 
+    def test_get_with_list_top_level(self):
+        d = {'a': [1, 2, 3],
+             'b': 'a',
+             'c': [1, (inc, 1)],
+             'd': [(sum, 'a')],
+             'e': ['a', 'b'],
+             'f': [[[(sum, 'a'), 'c'], (sum, 'b')], 2]}
+        assert self.get(d, 'a') == [1, 2, 3]
+        assert self.get(d, 'b') == [1, 2, 3]
+        assert self.get(d, 'c') == [1, 2]
+        assert self.get(d, 'd') == [6]
+        assert self.get(d, 'e') == [[1, 2, 3], [1, 2, 3]]
+        assert self.get(d, 'f') == [[[6, [1, 2]], 6], 2]
+
     def test_get_with_nested_list(self):
         d = {'x': 1, 'y': 2, 'z': (sum, ['x', 'y'])}
 
@@ -110,6 +122,6 @@ def test_get_stack_limit(self):
                 assert str(e).startswith('Found no accessible jobs in dask')
         else:
             msg = 'dask with infinite cycle should have raised an exception.'
-            self.assertTrue(False, msg=msg)
+            assert False, msg
 
         assert self.get(d, 'x4999') == 4999