Repository: mars-project/mars

Issue:
[BUG] Deleting broadcasted meta may cause item in kvstore be deleted multiple times
**Describe the bug**
Deleting broadcasted meta may cause item in kvstore be deleted multiple times.

**To Reproduce**
In multiple worker settings, error reported like

```
Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 716, in gevent._greenlet.Greenlet.run
  File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
    cpdef object fire_run(self):
  File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
    with self.lock:
  File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
    raise
  File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
    res = actor.on_receive(message_ctx.message)
  File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
    cpdef on_receive(self, message):
  File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
    return getattr(self, method)(*args, **kwargs)
  File "/home/admin/wenjun.swj/mars/mars/scheduler/kvstore.py", line 54, in delete
    return self._store.delete(key, dir=dir, recursive=recursive)
  File "/home/admin/wenjun.swj/mars/mars/kvstore.py", line 265, in delete
    raise KeyError(key)
KeyError: '/sessions/c3c9d252-09bc-11e9-bae9-97dedca03eb9/chunks/6f02ffa9ca5cf0b82362b37418d29ccc'
2018-12-27T10:17:46Z <Greenlet "Greenlet-0" at 0x7fed5e0f6268: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0x7fed5f8bbf48>> failed with KeyError
```



Discussion:

Resolving patch:
diff --git a/mars/scheduler/chunkmeta.py b/mars/scheduler/chunkmeta.py
--- a/mars/scheduler/chunkmeta.py
+++ b/mars/scheduler/chunkmeta.py
@@ -265,6 +265,9 @@ def delete_meta(self, session_id, chunk_key):
         query_key = (session_id, chunk_key)
         try:
             del self._meta_store[query_key]
+            if self._kv_store_ref is not None:
+                self._kv_store_ref.delete('/sessions/%s/chunks/%s' % (session_id, chunk_key),
+                                          recursive=True, _tell=True, _wait=False)
         except KeyError:
             pass
         try:
@@ -273,15 +276,10 @@ def delete_meta(self, session_id, chunk_key):
             pass
 
         # broadcast deletion into pre-determined destinations
-        futures = []
         if query_key in self._meta_broadcasts:
             for dest in self._meta_broadcasts[query_key]:
-                futures.append(self.ctx.actor_ref(self.default_name(), address=dest) \
-                               .delete_meta(session_id, chunk_key, _wait=False, _tell=True))
-        if self._kv_store_ref is not None:
-            futures.append(self._kv_store_ref.delete('/sessions/%s/chunks/%s' % (session_id, chunk_key),
-                                                     recursive=True, _tell=True, _wait=False))
-        [f.result() for f in futures]
+                self.ctx.actor_ref(self.default_name(), address=dest) \
+                    .delete_meta(session_id, chunk_key, _wait=False, _tell=True)
 
     def batch_delete_meta(self, session_id, chunk_keys):
         """


Test patch: