Repository: dask/distributed

Issue:
`performance_report` fails to write out if a worker has died during the compute
I had a (relatively) large compute running a few times today, where a worker would die off part way.

The computation finished and returned successfully, but I was presented with an error message as my code attempted to step outside of the `with performance_report():` block, as below:

<details>

```python
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-9-9dd6de6bd4c3> in <module>
     10         optimize_graph=False
     11     )
---> 12     print(df)

~/.conda/envs/datacompyler/lib/python3.7/site-packages/distributed/client.py in __exit__(self, typ, value, traceback)
   4618         except Exception:
   4619             code = ""
-> 4620         get_client().sync(self.__aexit__, type, value, traceback, code=code)
   4621 
   4622 

~/.conda/envs/datacompyler/lib/python3.7/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    778         else:
    779             return sync(
--> 780                 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
    781             )
    782 

~/.conda/envs/datacompyler/lib/python3.7/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    346     if error[0]:
    347         typ, exc, tb = error[0]
--> 348         raise exc.with_traceback(tb)
    349     else:
    350         return result[0]

~/.conda/envs/datacompyler/lib/python3.7/site-packages/distributed/utils.py in f()
    330             if callback_timeout is not None:
    331                 future = asyncio.wait_for(future, callback_timeout)
--> 332             result[0] = yield future
    333         except Exception as exc:
    334             error[0] = sys.exc_info()

~/.conda/envs/datacompyler/lib/python3.7/site-packages/tornado/gen.py in run(self)
    733 
    734                     try:
--> 735                         value = future.result()
    736                     except Exception:
    737                         exc_info = sys.exc_info()

~/.conda/envs/datacompyler/lib/python3.7/site-packages/distributed/client.py in __aexit__(self, typ, value, traceback, code)
   4604                 code = ""
   4605         data = await get_client().scheduler.performance_report(
-> 4606             start=self.start, code=code
   4607         )
   4608         with open(self.filename, "w") as f:

~/.conda/envs/datacompyler/lib/python3.7/site-packages/distributed/core.py in send_recv_from_rpc(**kwargs)
    755             name, comm.name = comm.name, "ConnectionPool." + key
    756             try:
--> 757                 result = await send_recv(comm=comm, op=key, **kwargs)
    758             finally:
    759                 self.pool.reuse(self.addr, comm)

~/.conda/envs/datacompyler/lib/python3.7/site-packages/distributed/core.py in send_recv(comm, reply, serializers, deserializers, **kwargs)
    554         if comm.deserialize:
    555             typ, exc, tb = clean_exception(**response)
--> 556             raise exc.with_traceback(tb)
    557         else:
    558             raise Exception(response["text"])

/opt/anaconda3/lib/python3.7/site-packages/distributed/core.py in handle_comm()
    410                             result = asyncio.ensure_future(result)
    411                             self._ongoing_coroutines.add(result)
--> 412                             result = await result
    413                     except (CommClosedError, CancelledError) as e:
    414                         if self.status == "running":

/opt/anaconda3/lib/python3.7/site-packages/distributed/scheduler.py in performance_report()
   5024 
   5025         bandwidth_workers = BandwidthWorkers(self, sizing_mode="stretch_both")
-> 5026         bandwidth_workers.update()
   5027         bandwidth_types = BandwidthTypes(self, sizing_mode="stretch_both")
   5028         bandwidth_types.update()

/opt/anaconda3/lib/python3.7/site-packages/bokeh/core/property/validation.py in func()
     95     def func(*args, **kwargs):
     96         with validate(False):
---> 97             return input_function(*args, **kwargs)
     98     return func
     99 

/opt/anaconda3/lib/python3.7/site-packages/distributed/dashboard/components/scheduler.py in update()
    419                     return address
    420 
--> 421             x, y, value = zip(*[(name(a), name(b), c) for (a, b), c in bw.items()])
    422 
    423             self.color_map.high = max(value)

/opt/anaconda3/lib/python3.7/site-packages/distributed/dashboard/components/scheduler.py in <listcomp>()
    419                     return address
    420 
--> 421             x, y, value = zip(*[(name(a), name(b), c) for (a, b), c in bw.items()])
    422 
    423             self.color_map.high = max(value)

/opt/anaconda3/lib/python3.7/site-packages/distributed/dashboard/components/scheduler.py in name()
    413 
    414             def name(address):
--> 415                 ws = self.scheduler.workers[address]
    416                 if ws.name is not None:
    417                     return str(ws.name)

KeyError: 'tcp://127.0.0.1:36457'
```

</details>

`tcp://127.0.0.1:36457` was not a worker that I could see as existing on the `LocalCluster` after the compute had finished, and it appeared that the total number of worker processes left idle at the end was less than what I had initially stood the cluster with.

I think the performance report is attempting to gather report data on the worker process that no longer exists?


Discussion:
Thank you for the excellent bug report @KrishanBhasin .  I think that your guess is correct, that we're trying to get the name of a worker that no longer exists.  I think case I think that we have two options:

1. Don't include the worker in the report
2.  Include the worker and just use the address rather than the name (which is the default behavior if the worker has no name anyway)

I'm in favor of option 2.  In that case we should be able to resolve this by doing a try-except around getting the worker in line 415 there.  Is this something that you would like to contribute?

Resolving patch:
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -412,7 +412,10 @@ def update(self):
                 return
 
             def name(address):
-                ws = self.scheduler.workers[address]
+                try:
+                    ws = self.scheduler.workers[address]
+                except KeyError:
+                    return address
                 if ws.name is not None:
                     return str(ws.name)
                 else:


Test patch: