Repository: dask/distributed

Issue:
Variable delete raises KeyError
It seems that there is an unhandled KeyError in the scheduler variable delete logic.

If using the asynchronous API only visible in the scheduler logs but it does mean that Variables will be "leaking" and staying around instead of being deleted.

It happens regardless of whether you set a value.

Shortest reproduction I can provide is as follows:
```python
from distributed import Client
from distributed import Variable


def main():
    Client()
    Variable().delete()


if __name__ == '__main__':
    main()
```

```python
tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <tornado.platform.asyncio.AsyncIOLoop object at 0x7f8fe9265be0>>, <Task finished coro=<VariableExtension.delete() done, defined at ./venv/lib/python3.6/site-packages/distributed/variable.py:101> exception=KeyError('variable-68a51209410248fa9f69f84b25f48343',)>)
Traceback (most recent call last):
  File "./venv/lib/python3.6/site-packages/tornado/ioloop.py", line 743, in _run_callback
    ret = callback()
  File "./venv/lib/python3.6/site-packages/tornado/ioloop.py", line 767, in _discard_future_result
    future.result()
  File "./venv/lib/python3.6/site-packages/distributed/variable.py", line 110, in delete
    del self.waiting_conditions[name]
KeyError: 'variable-68a51209410248fa9f69f84b25f48343'
```

Python 3.6.10
Reproduced with distributed 2.9.3 and 2.10.0


Discussion:
@jacobtomlinson based on your interests this may be something you would find educational.

Resolving patch:
diff --git a/distributed/variable.py b/distributed/variable.py
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -113,8 +113,10 @@ async def delete(self, stream=None, name=None, client=None):
             else:
                 if old["type"] == "Future":
                     await self.release(old["value"], name)
-            del self.waiting_conditions[name]
-            del self.variables[name]
+            with ignoring(KeyError):
+                del self.waiting_conditions[name]
+            with ignoring(KeyError):
+                del self.variables[name]
 
 
 class Variable:


Test patch:
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -2,6 +2,7 @@
 import random
 from time import sleep
 import sys
+import logging
 
 import pytest
 from tornado import gen
@@ -12,6 +13,7 @@
 from distributed.compatibility import WINDOWS
 from distributed.utils_test import gen_cluster, inc, div
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
+from distributed.utils_test import captured_logger
 
 
 @gen_cluster(client=True)
@@ -39,6 +41,17 @@ def test_variable(c, s, a, b):
         assert time() < start + 5
 
 
+@gen_cluster(client=True)
+async def test_delete_unset_variable(c, s, a, b):
+    x = Variable()
+    assert x.client is c
+    with captured_logger(logging.getLogger("distributed.utils")) as logger:
+        x.delete()
+        await c.close()
+    text = logger.getvalue()
+    assert "KeyError" not in text
+
+
 @gen_cluster(client=True)
 def test_queue_with_data(c, s, a, b):
     x = Variable("x")