Repository: dask/distributed

Issue:
Calling get() on a variable returns a future which does not have the latest state
Simple. Submit a future and add it to a variable. Then, from another client "peek" at the variable, i.e. don't call result() just look at the state and see if it is finished by calling variable.get()

Behavior: sometimes the state is fresh while other times it takes long to actually get the latest state of the future. 

By calling result on the future everything is fine. However I can't block  and need to return. Only when the future is finished can the caller block to get the result.

`   def inc(self, x):
        return x + 1

    def dummy(self, x):
        pass

    def test_variables_2(self):
        with Client(cfg.current_config.DISTRIBUTED_SCHEDULER) as client:
            L = client.map(self.inc, range(1000))
            f = client.submit(self.dummy, L)

            v = Variable(name="var", client=client)
            v.set(f)

            client.compute(f)
    
        while True:
            with Client(cfg.current_config.DISTRIBUTED_SCHEDULER) as client:
                v = Variable(name="var", client=client)
                f = v.get()

                if f.status == 'finished':
                    break
                else:
                    print('not finished')
                    time.sleep(1)``
`


Discussion:
When the client first creates a future it doesn't yet have the status of that future.  It may take a few milliseconds as the client asks the scheduler for more information.  In your example you immediately check the status of the future directly after getting it you'll find that the client doesn't yet know the status.  If you wait a few milliseconds you'll find that everything is fine.  

Here is an example that tests this timing by waiting on the future.

```python
In [1]: from dask.distributed import Client, Variable, wait

In [2]: import time

In [3]: def inc(x):
   ...:     return x + 1
   ...: 
   ...: def dummy(x):
   ...:     pass
   ...: 
   ...: def test_variables_2():
   ...:     with Client('localhost:8786') as client:
   ...:         L = client.map(inc, range(1000))
   ...:         f = client.submit(dummy, L)
   ...: 
   ...:         v = Variable(name="var", client=client)
   ...:         v.set(f)
   ...: 
   ...:         client.compute(f)
   ...: 
   ...:     while True:
   ...:         with Client('localhost:8786') as client:
   ...:             v = Variable(name="var", client=client)
   ...:             f = v.get()
   ...:             start = time.time()
   ...:             wait(f)
   ...:             end = time.time()
   ...:             print("Duration", end - start)
   ...:             print("Status", f.status)
   ...:             

In [4]: test_variables_2()
Duration 0.3927021026611328
Status finished
Duration 0.0009415149688720703
Status finished
Duration 0.0009622573852539062
Status finished
Duration 0.004920244216918945
Status finished
Duration 0.0009350776672363281
Status finished
Duration 0.0010135173797607422
Status finished
Duration 0.0009930133819580078
Status finished
Duration 0.0025153160095214844
Status finished
Duration 0.0008144378662109375
Status finished
Duration 0.003805875778198242
Status finished
Duration 0.002122163772583008
Status finished
Duration 0.002368450164794922
Status finished
Duration 0.0009746551513671875
Status finished
Duration 0.0007257461547851562
Status finished
Duration 0.0020928382873535156
Status finished
Duration 0.002156496047973633
Status finished
```
thanks! waiting a bit does indeed bring the latest state. I will test a bit more to make sure.
We could consider sending the current status of the future when the user gets the future.  This would save a millisecond or two (which is non-trivial for some applications) and would avoid tricky situations like this.
I agree. I would expect get() to wait until the future status is retrieved and then return. I am now waiting 0.1 seconds after the get() and everything works fine. Are there scenarios where the scheduler response can be delayed significantly? 

If waiting a defined amount of time is the suggested solution we can easily close this issue. 

Resolving patch:
diff --git a/distributed/client.py b/distributed/client.py
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -94,7 +94,7 @@ class Future(WrappedKey):
     _cb_executor = None
     _cb_executor_pid = None
 
-    def __init__(self, key, client, inform=False):
+    def __init__(self, key, client, inform=False, state=None):
         self.key = key
         self._cleared = False
         tkey = tokey(key)
@@ -112,6 +112,14 @@ def __init__(self, key, client, inform=False):
                                             'keys': [tokey(key)],
                                             'client': self.client.id})
 
+        if state is not None:
+            try:
+                handler = self.client._state_handlers[state]
+            except KeyError:
+                pass
+            else:
+                handler(key=key)
+
     @property
     def executor(self):
         return self.client
@@ -469,6 +477,12 @@ def __init__(self, address=None, loop=None, timeout=5,
             'error': self._handle_error
         }
 
+        self._state_handlers = {
+            'memory': self._handle_key_in_memory,
+            'lost': self._handle_lost_data,
+            'erred': self._handle_task_erred
+        }
+
         super(Client, self).__init__(connection_args=self.connection_args,
                                      io_loop=self.loop)
 
diff --git a/distributed/queues.py b/distributed/queues.py
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -7,6 +7,11 @@
 from tornado import gen
 import tornado.queues
 
+try:
+    from cytoolz import assoc
+except ImportError:
+    from toolz import assoc
+
 from .client import Future, _get_global_client, Client
 from .utils import tokey, sync
 
@@ -74,9 +79,40 @@ def future_release(self, name=None, key=None, client=None):
             del self.future_refcount[name, key]
 
     @gen.coroutine
-    def get(self, stream=None, name=None, client=None, timeout=None):
-        record = yield self.queues[name].get(timeout=timeout)
-        raise gen.Return(record)
+    def get(self, stream=None, name=None, client=None, timeout=None,
+            batch=False):
+        def process(record):
+            """ Add task status if known """
+            if record['type'] == 'Future':
+                try:
+                    state = self.scheduler.task_state[record['value']]
+                except KeyError:
+                    state = 'lost'
+                return assoc(record, 'state', state)
+            else:
+                return record
+
+        if batch:
+            q = self.queues[name]
+            out = []
+            if batch is True:
+                while not q.empty():
+                    record = yield q.get()
+                    out.append(record)
+            else:
+                if timeout is not None:
+                    msg = ("Dask queues don't support simultaneous use of "
+                           "integer batch sizes and timeouts")
+                    raise NotImplementedError(msg)
+                for i in range(batch):
+                    record = yield q.get()
+                    out.append(record)
+            out = [process(o) for o in out]
+            raise gen.Return(out)
+        else:
+            record = yield self.queues[name].get(timeout=timeout)
+            record = process(record)
+            raise gen.Return(record)
 
     def qsize(self, stream=None, name=None, client=None):
         return self.queues[name].qsize()
@@ -143,25 +179,48 @@ def put(self, value, timeout=None):
         """ Put data into the queue """
         return self.client.sync(self._put, value, timeout=timeout)
 
-    def get(self, timeout=None):
-        """ Get data from the queue """
-        return self.client.sync(self._get, timeout=timeout)
+    def get(self, timeout=None, batch=False):
+        """ Get data from the queue
+
+        Parameters
+        ----------
+        timeout: Number (optional)
+            Time in seconds to wait before timing out
+        batch: boolean, int (optional)
+            If True then return all elements currently waiting in the queue.
+            If an integer than return that many elements from the queue
+            If False (default) then return one item at a time
+         """
+        return self.client.sync(self._get, timeout=timeout, batch=batch)
 
     def qsize(self):
         """ Current number of elements in the queue """
         return self.client.sync(self._qsize)
 
     @gen.coroutine
-    def _get(self, timeout=None):
-        d = yield self.client.scheduler.queue_get(timeout=timeout, name=self.name)
-        if d['type'] == 'Future':
-            value = Future(d['value'], self.client, inform=True)
-            self.client._send_to_scheduler({'op': 'queue-future-release',
-                                            'name': self.name,
-                                            'key': d['value']})
+    def _get(self, timeout=None, batch=False):
+        resp = yield self.client.scheduler.queue_get(timeout=timeout,
+                                                     name=self.name,
+                                                     batch=batch)
+
+        def process(d):
+            if d['type'] == 'Future':
+                value = Future(d['value'], self.client, inform=True,
+                               state=d['state'])
+                self.client._send_to_scheduler({'op': 'queue-future-release',
+                                                'name': self.name,
+                                                'key': d['value']})
+            else:
+                value = d['value']
+
+            return value
+
+        if batch is False:
+            result = process(resp)
         else:
-            value = d['value']
-        raise gen.Return(value)
+            result = list(map(process, resp))
+
+        raise gen.Return(result)
 
     @gen.coroutine
     def _qsize(self):
diff --git a/distributed/variable.py b/distributed/variable.py
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -7,6 +7,11 @@
 from tornado import gen
 import tornado.locks
 
+try:
+    from cytoolz import merge
+except ImportError:
+    from toolz import merge
+
 from .client import Future, _get_global_client, Client
 from .metrics import time
 from .utils import tokey, log_errors
@@ -82,9 +87,14 @@ def get(self, stream=None, name=None, client=None, timeout=None):
             yield self.started.wait(timeout=left)
         record = self.variables[name]
         if record['type'] == 'Future':
+            key = record['value']
             token = uuid.uuid4().hex
-            record['token'] = token
-            self.waiting[record['value'], name].add(token)
+            try:
+                state = self.scheduler.task_state[key]
+            except KeyError:
+                state = 'lost'
+            record = merge(record, {'token': token, 'state': state})
+            self.waiting[key, name].add(token)
         raise gen.Return(record)
 
     @gen.coroutine
@@ -161,7 +171,7 @@ def _get(self, timeout=None):
                                                      name=self.name,
                                                      client=self.client.id)
         if d['type'] == 'Future':
-            value = Future(d['value'], self.client, inform=True)
+            value = Future(d['value'], self.client, inform=True, state=d['state'])
             self.client._send_to_scheduler({'op': 'variable-future-release',
                                             'name': self.name,
                                             'key': d['value'],


Test patch:
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+from datetime import timedelta
 from operator import add
 from time import sleep
 import sys
@@ -8,10 +9,10 @@
 from toolz import take
 from tornado import gen
 
-from distributed import Client, Queue, Nanny, worker_client
+from distributed import Client, Queue, Nanny, worker_client, wait
 from distributed.metrics import time
 from distributed.utils_test import (gen_cluster, inc, loop, cluster, slowinc,
-                                    slow)
+                                    slow, div)
 
 
 @gen_cluster(client=True)
@@ -169,3 +170,60 @@ def test_same_futures(c, s, a, b):
     while s.wants_what['queue-x']:
         yield gen.sleep(0.01)
         assert time() - start < 2
+
+
+@gen_cluster(client=True)
+def test_get_many(c, s, a, b):
+    x = yield Queue('x')
+    xx = yield Queue('x')
+
+    yield x.put(1)
+    yield x.put(2)
+    yield x.put(3)
+
+    data = yield xx.get(batch=True)
+    assert data == [1, 2, 3]
+
+    yield x.put(1)
+    yield x.put(2)
+    yield x.put(3)
+
+    data = yield xx.get(batch=2)
+    assert data == [1, 2]
+
+    with pytest.raises(gen.TimeoutError):
+        data = yield gen.with_timeout(timedelta(seconds=0.100),
+                                      xx.get(batch=2))
+
+
+@gen_cluster(client=True)
+def test_Future_knows_status_immediately(c, s, a, b):
+    x = yield c.scatter(123)
+    q = yield Queue('q')
+    yield q.put(x)
+
+    c2 = yield Client(s.address, asynchronous=True)
+    q2 = yield Queue('q', client=c2)
+    future = yield q2.get()
+    assert future.status == 'finished'
+
+    x = c.submit(div, 1, 0)
+    yield wait(x)
+    yield q.put(x)
+
+    future2 = yield q2.get()
+    assert future2.status == 'error'
+    with pytest.raises(Exception):
+        yield future2
+
+    start = time()
+    while True:  # we learn about the true error eventually
+        try:
+            yield future2
+        except ZeroDivisionError:
+            break
+        except Exception:
+            assert time() < start + 5
+            yield gen.sleep(0.05)
+
+    yield c2.shutdown()
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -8,10 +8,10 @@
 from toolz import take
 from tornado import gen
 
-from distributed import Client, Variable, worker_client, Nanny
+from distributed import Client, Variable, worker_client, Nanny, wait
 from distributed.metrics import time
 from distributed.utils_test import (gen_cluster, inc, loop, cluster, slowinc,
-                                    slow)
+                                    slow, div)
 
 
 @gen_cluster(client=True)
@@ -172,3 +172,36 @@ def f(i):
         yield gen.sleep(0.01)
         if not time() - start < 2:
             import pdb; pdb.set_trace()
+
+
+@gen_cluster(client=True)
+def test_Future_knows_status_immediately(c, s, a, b):
+    x = yield c.scatter(123)
+    v = Variable('x')
+    yield v.set(x)
+
+    c2 = yield Client(s.address, asynchronous=True)
+    v2 = Variable('x', client=c2)
+    future = yield v2.get()
+    assert future.status == 'finished'
+
+    x = c.submit(div, 1, 0)
+    yield wait(x)
+    yield v.set(x)
+
+    future2 = yield v2.get()
+    assert future2.status == 'error'
+    with pytest.raises(Exception):
+        yield future2
+
+    start = time()
+    while True:  # we learn about the true error eventually
+        try:
+            yield future2
+        except ZeroDivisionError:
+            break
+        except Exception:
+            assert time() < start + 5
+            yield gen.sleep(0.05)
+
+    yield c2.shutdown()