Repository: dask/distributed

Issue:
Executor raises StreamClosedError when Scheduler is not present
I get a rather obscure error if scheduler is not present at the URI  on master on OSX. I cannot think of a reason why this needs to be handled right now except for clarity. 

``` python
In [1]: from distributed import Executor

In [2]: a = Executor('localhost:8786') # scheduler is not present at this URI
distributed.utils - ERROR - Stream is closed
Traceback (most recent call last):
---------------------------------------------------------------------------
StreamClosedError                         Traceback (most recent call last)
<ipython-input-2-bdc308648e14> in <module>()
----> 1 a = Executor('172.17.0.5:8786')

workspace/venv/lib/python2.7/site-packages/distributed/executor.pyc in __init__(self, address, start, loop)
    216 
    217         if start:
--> 218             self.start()
    219 
    220     def start(self, **kwargs):

workspace/venv/lib/python2.7/site-packages/distributed/executor.pyc in start(self, **kwargs)
    231         while not self.loop._running:
    232             sleep(0.001)
--> 233         sync(self.loop, self._start, **kwargs)
    234 
    235     def _send_to_scheduler(self, msg):

workspace/venv/lib/python2.7/site-packages/distributed/utils.pyc in sync(loop, func, *args, **kwargs)
    103     e.wait()
    104     if error[0]:
--> 105         raise result[0]
    106     else:
    107         return result[0]

StreamClosedError: Stream is closed
```



Discussion:

Resolving patch:
diff --git a/distributed/core.py b/distributed/core.py
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -277,12 +277,13 @@ class rpc(object):
 
     >>> remote.close_streams()  # doctest: +SKIP
     """
-    def __init__(self, stream=None, ip=None, port=None):
+    def __init__(self, stream=None, ip=None, port=None, timeout=3):
         self.streams = dict()
         if stream:
             self.streams[stream] = True
         self.ip = ip
         self.port = port
+        self.timeout = timeout
 
     @gen.coroutine
     def live_stream(self):
@@ -311,7 +312,7 @@ def live_stream(self):
             if open:
                 break
         if not open or stream.closed():
-            stream = yield connect(self.ip, self.port)
+            stream = yield connect(self.ip, self.port, timeout=self.timeout)
             self.streams[stream] = True
         for s in to_clear:
             del self.streams[s]
@@ -333,14 +334,14 @@ def _(**kwargs):
         return _
 
 
-def coerce_to_rpc(o):
+def coerce_to_rpc(o, **kwargs):
     if isinstance(o, tuple):
-        return rpc(ip=o[0], port=o[1])
+        return rpc(ip=o[0], port=o[1], **kwargs)
     if isinstance(o, str):
         ip, port = o.split(':')
-        return rpc(ip=ip, port=int(port))
+        return rpc(ip=ip, port=int(port), **kwargs)
     elif isinstance(o, IOStream):
-        return rpc(stream=o)
+        return rpc(stream=o, **kwargs)
     elif isinstance(o, rpc):
         return o
     else:
diff --git a/distributed/executor.py b/distributed/executor.py
--- a/distributed/executor.py
+++ b/distributed/executor.py
@@ -16,7 +16,7 @@
 from dask.compatibility import apply
 from toolz import first, groupby, merge
 from tornado import gen
-from tornado.gen import Return
+from tornado.gen import Return, TimeoutError
 from tornado.locks import Event
 from tornado.ioloop import IOLoop, PeriodicCallback
 from tornado.iostream import StreamClosedError, IOStream
@@ -207,7 +207,7 @@ class Executor(object):
     --------
     distributed.scheduler.Scheduler: Internal scheduler
     """
-    def __init__(self, address, start=True, loop=None):
+    def __init__(self, address, start=True, loop=None, timeout=3):
         self.futures = dict()
         self.refcount = defaultdict(lambda: 0)
         self.loop = loop or IOLoop()
@@ -215,7 +215,7 @@ def __init__(self, address, start=True, loop=None):
         self._start_arg = address
 
         if start:
-            self.start()
+            self.start(timeout=timeout)
 
     def start(self, **kwargs):
         """ Start scheduler running in separate thread """
@@ -241,7 +241,7 @@ def _send_to_scheduler(self, msg):
             raise NotImplementedError()
 
     @gen.coroutine
-    def _start(self, **kwargs):
+    def _start(self, timeout=3, **kwargs):
         if isinstance(self._start_arg, Scheduler):
             self.scheduler = self._start_arg
             self.center = self._start_arg.center
@@ -249,8 +249,11 @@ def _start(self, **kwargs):
             ip, port = tuple(self._start_arg.split(':'))
             self._start_arg = (ip, int(port))
         if isinstance(self._start_arg, tuple):
-            r = coerce_to_rpc(self._start_arg)
-            ident = yield r.identity()
+            r = coerce_to_rpc(self._start_arg, timeout=timeout)
+            try:
+                ident = yield r.identity()
+            except (StreamClosedError, OSError):
+                raise IOError("Could not connect to %s:%d" % self._start_arg)
             if ident['type'] == 'Center':
                 self.center = r
                 self.scheduler = Scheduler(self.center, loop=self.loop,


Test patch:
diff --git a/distributed/tests/test_executor.py b/distributed/tests/test_executor.py
--- a/distributed/tests/test_executor.py
+++ b/distributed/tests/test_executor.py
@@ -1528,3 +1528,15 @@ def test_allow_restrictions(s, a, b):
 
     with pytest.raises(TypeError):
         e.map(inc, [20], workers='127.0.0.1', allow_other_workers='Hello!')
+
+
+def test_bad_address():
+    try:
+        Executor('123.123.123.123:1234', timeout=0.1)
+    except (IOError, gen.TimeoutError) as e:
+        assert "connect" in str(e).lower()
+
+    try:
+        Executor('127.0.0.1:1234', timeout=0.1)
+    except (IOError, gen.TimeoutError) as e:
+        assert "connect" in str(e).lower()