Repository: ipython/ipython

Issue:
AsyncResult.wait(0) can hang waiting for the client to get results?
I have some code that runs multiple maps on a load balanced view. I run the code on a remote ipengine that I'm connected to via ssh; the client is running locally in an ipython notebook. The individual tasks take some time to run, perhaps a minute or two each.

It appears that AsyncResult instances can end up in a situation where all outstanding jobs for the are finished, but where the outputs are not actually ready for the client yet. Thus _ready is not set, and somehow wait(0) ends up at self._wait_for_outputs(10) (line 161 of asyncresult.py). This hangs for ten seconds owing to the ten second timeout.

As a result, any function which uses wait(0) will hang for ten seconds before returning, including AsyncResult.progress.

While I'm not exactly sure _why_ this is happening, it seems like it can be solved reasonably easily by changing 10 to timeout, thus causing the timeout there to honor the timeout given to wait.



Discussion:
I don't think it's possible for `_wait_for_outputs` to be called if `_ready` is not set, since it is inside a `if self._ready:` block.

Can you post code to reproduce this?  I have seen it a while ago, but can't reproduce it well enough to actually test and find a fix.

I agree that relaying the timeout would be a decent band-aid, but there's a more serious bug causing this that I would like to actually find and fix.


Resolving patch:
diff --git a/IPython/parallel/client/asyncresult.py b/IPython/parallel/client/asyncresult.py
--- a/IPython/parallel/client/asyncresult.py
+++ b/IPython/parallel/client/asyncresult.py
@@ -82,6 +82,7 @@ def __init__(self, client, msg_ids, fname='unknown', targets=None, tracker=None)
         self._targets = targets
         self._tracker = tracker
         self._ready = False
+        self._outputs_ready = False
         self._success = None
         self._metadata = [ self._client.metadata.get(id) for id in self.msg_ids ]
         if len(msg_ids) == 1:
@@ -134,6 +135,9 @@ def ready(self):
         """Return whether the call has completed."""
         if not self._ready:
             self.wait(0)
+        elif not self._outputs_ready:
+            self._wait_for_outputs(0)
+        
         return self._ready
 
     def wait(self, timeout=-1):
@@ -142,6 +146,7 @@ def wait(self, timeout=-1):
         This method always returns None.
         """
         if self._ready:
+            self._wait_for_outputs(timeout)
             return
         self._ready = self._client.wait(self.msg_ids, timeout)
         if self._ready:
@@ -161,8 +166,10 @@ def wait(self, timeout=-1):
             else:
                 self._success = True
             finally:
-                
-                self._wait_for_outputs(10)
+                if timeout is None or timeout < 0:
+                    # cutoff infinite wait at 10s
+                    timeout = 10
+                self._wait_for_outputs(timeout)
 
 
     def successful(self):
@@ -251,6 +258,7 @@ def __getitem__(self, key):
             return error.collect_exceptions(self._result[key], self._fname)
         elif isinstance(key, basestring):
             # metadata proxy *does not* require that results are done
+            self.wait(0)
             values = [ md[key] for md in self._metadata ]
             if self._single_result:
                 return values[0]
@@ -377,11 +385,13 @@ def wall_time(self):
         """
         return self.timedelta(self.submitted, self.received)
     
-    def wait_interactive(self, interval=1., timeout=None):
+    def wait_interactive(self, interval=1., timeout=-1):
         """interactive wait, printing progress at regular intervals"""
+        if timeout is None:
+            timeout = -1
         N = len(self)
         tic = time.time()
-        while not self.ready() and (timeout is None or time.time() - tic <= timeout):
+        while not self.ready() and (timeout < 0 or time.time() - tic <= timeout):
             self.wait(interval)
             clear_output()
             print("%4i/%i tasks finished after %4i s" % (self.progress, N, self.elapsed), end="")
@@ -433,13 +443,21 @@ def _display_single_result(self):
     def _wait_for_outputs(self, timeout=-1):
         """wait for the 'status=idle' message that indicates we have all outputs
         """
-        if not self._success:
+        if self._outputs_ready or not self._success:
             # don't wait on errors
             return
+        
+        # cast None to -1 for infinite timeout
+        if timeout is None:
+            timeout = -1
+        
         tic = time.time()
-        while not all(md['outputs_ready'] for md in self._metadata):
+        self._client._flush_iopub(self._client._iopub_socket)
+        self._outputs_ready = all(md['outputs_ready'] for md in self._metadata)
+        while not self._outputs_ready:
             time.sleep(0.01)
             self._client._flush_iopub(self._client._iopub_socket)
+            self._outputs_ready = all(md['outputs_ready'] for md in self._metadata)
             if timeout >= 0 and time.time() > tic + timeout:
                 break
     
@@ -643,9 +661,9 @@ class AsyncHubResult(AsyncResult):
     so use `AsyncHubResult.wait()` sparingly.
     """
 
-    def _wait_for_outputs(self, timeout=None):
+    def _wait_for_outputs(self, timeout=-1):
         """no-op, because HubResults are never incomplete"""
-        return
+        self._outputs_ready = True
     
     def wait(self, timeout=-1):
         """wait for result to complete."""


Test patch:
diff --git a/IPython/parallel/tests/test_asyncresult.py b/IPython/parallel/tests/test_asyncresult.py
--- a/IPython/parallel/tests/test_asyncresult.py
+++ b/IPython/parallel/tests/test_asyncresult.py
@@ -18,6 +18,8 @@
 
 import time
 
+import nose.tools as nt
+
 from IPython.utils.io import capture_output
 
 from IPython.parallel.error import TimeoutError
@@ -263,5 +265,30 @@ def test_display_empty_streams_engine(self):
             ar.display_outputs('engine')
         self.assertEqual(io.stderr, '')
         self.assertEqual(io.stdout, '')
+    
+    def test_await_data(self):
+        """asking for ar.data flushes outputs"""
+        self.minimum_engines(1)
         
+        v = self.client[-1]
+        ar = v.execute('\n'.join([
+            "import time",
+            "from IPython.zmq.datapub import publish_data",
+            "for i in range(5):",
+            "    publish_data(dict(i=i))",
+            "    time.sleep(0.1)",
+        ]), block=False)
+        found = set()
+        tic = time.time()
+        # timeout after 10s
+        while time.time() <= tic + 10:
+            if ar.data:
+                found.add(ar.data['i'])
+                if ar.data['i'] == 4:
+                    break
+            time.sleep(0.05)
+        
+        ar.get(5)
+        nt.assert_in(4, found)
+        self.assertTrue(len(found) > 1, "should have seen data multiple times, but got: %s" % found)