Repository: dask/distributed

Issue:
WorkerState localdirectory not initialized
I have been seeing these tracebacks occassionally when using distributed (1.24.0)

	python3.5/site-packages/distributed/utils.py", line 277, in sync
		six.reraise(*error[0])
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/six.py", line 693, in reraise
		raise value
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/utils.py", line 262, in f
		result[0] = yield future
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1133, in run
		value = future.result()
	  File "./.conda/envs/xyz/lib/python3.5/asyncio/futures.py", line 294, in result
		raise self._exception
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1141, in run
		yielded = self.gen.throw(*exc_info)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/client.py", line 821, in _start
		yield self._ensure_connected(timeout=timeout)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1133, in run
		value = future.result()
	  File "./.conda/envs/xyz/lib/python3.5/asyncio/futures.py", line 294, in result
		raise self._exception
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1141, in run
		yielded = self.gen.throw(*exc_info)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/client.py", line 862, in _ensure_connected
		self._update_scheduler_info())
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1133, in run
		value = future.result()
	  File "./.conda/envs/xyz/lib/python3.5/asyncio/futures.py", line 294, in result
		raise self._exception
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1141, in run
		yielded = self.gen.throw(*exc_info)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/client.py", line 896, in _update_scheduler_info
		self._scheduler_identity = yield self.scheduler.identity()
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1133, in run
		value = future.result()
	  File "./.conda/envs/xyz/lib/python3.5/asyncio/futures.py", line 294, in result
		raise self._exception
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1141, in run
		yielded = self.gen.throw(*exc_info)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/core.py", line 574, in send_recv_from_rpc
		result = yield send_recv(comm=comm, op=key, **kwargs)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1133, in run
		value = future.result()
	  File "./.conda/envs/xyz/lib/python3.5/asyncio/futures.py", line 294, in result
		raise self._exception
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/tornado/gen.py", line 1147, in run
		yielded = self.gen.send(value)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/core.py", line 466, in send_recv
		six.reraise(*clean_exception(**response))
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/six.py", line 692, in reraise
		raise value.with_traceback(tb)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/core.py", line 337, in handle_comm
		result = handler(comm, **msg)
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/scheduler.py", line 1018, in identity
		for worker in self.workers.values()}}
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/scheduler.py", line 1018, in <dictcomp>
		for worker in self.workers.values()}}
	  File "./.conda/envs/xyz/lib/python3.5/site-packages/distributed/scheduler.py", line 251, in identity
		'local_directory': self.local_directory,
	AttributeError: local_directory

The `local_directory` slot is not initialized in `__init__`. Is this intentional, or an oversight?


Discussion:
Nice catch. I suspect that the problem is here:

https://github.com/dask/distributed/blob/e6b1ef5027c4914170af12882ab4528dce2e2ba3/distributed/scheduler.py#L1276-L1302

Notice that we create the `WorkerState` object, then there is a `yield` (during which anything can happen really, including the identity/heartbeat check) and then we finally assign `local_directory`.  The easist solution is probably to assign a dummy value like `None` or the empty string for that (and all other values) in the constructor.

Resolving patch:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -193,9 +193,10 @@ class WorkerState(object):
     # XXX need a state field to signal active/removed?
 
     __slots__ = (
+        'actors',
         'address',
         'has_what',
-        'info',
+        'last_seen',
         'local_directory',
         'memory_limit',
         'metrics',
@@ -206,30 +207,34 @@ class WorkerState(object):
         'pid',
         'processing',
         'resources',
-        'time_delay',
-        'used_resources',
         'services',
         'status',
-        'last_seen',
-        'actors',
+        'time_delay',
+        'used_resources',
     )
 
-    def __init__(self, worker, ncores, memory_limit, name=None):
-        self.address = worker
-        self.has_what = set()
-        self.memory_limit = memory_limit
+    def __init__(self, address=None, pid=0, name=None, ncores=0, memory_limit=0,
+                 local_directory=None, services=None):
+        self.address = address
+        self.pid = pid
         self.name = name
-        self.nbytes = 0
         self.ncores = ncores
+        self.memory_limit = memory_limit
+        self.local_directory = local_directory
+        self.services = services or {}
+
+        self.status = 'running'
+        self.nbytes = 0
         self.occupancy = 0
-        self.pid = 0
-        self.processing = dict()
-        self.resources = {}
-        self.used_resources = {}
+        self.metrics = {}
         self.last_seen = 0
-        self.services = {}
+        self.time_delay = 0
+
         self.actors = set()
-        self.metrics = {}
+        self.has_what = set()
+        self.processing = {}
+        self.resources = {}
+        self.used_resources = {}
 
     @property
     def host(self):
@@ -1277,9 +1282,15 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
             if ws is not None:
                 raise ValueError("Worker already exists %s" % address)
 
-            ws = WorkerState(address, ncores, memory_limit, name)
-            ws.status = 'running'
-            self.workers[address] = ws
+            self.workers[address] = ws = WorkerState(
+                    address=address,
+                    pid=pid,
+                    ncores=ncores,
+                    memory_limit=memory_limit,
+                    name=name,
+                    local_directory=local_directory,
+                    services=services
+            )
 
             if name in self.aliases:
                 msg = {'status': 'error',
@@ -1296,10 +1307,6 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
 
             self.total_ncores += ncores
             self.aliases[name] = address
-            ws.name = name
-            ws.pid = pid
-            ws.services = services
-            ws.local_directory = local_directory
 
             response = self.heartbeat_worker(address=address,
                                              resolve_address=resolve_address,


Test patch:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3714,6 +3714,8 @@ def test_write_scheduler_file(c):
 
 
 def test_get_versions(c):
+    requests = pytest.importorskip('requests')
+
     v = c.get_versions()
     assert v['scheduler'] is not None
     assert v['client'] is not None
@@ -3726,7 +3728,6 @@ def test_get_versions(c):
     # that this does not raise
 
     v = c.get_versions(packages=['requests'])
-    import requests
     assert dict(v['client']['packages']['optional'])['requests'] == requests.__version__