# Buggy code
def delete_meta(self, session_id, chunk_key):
    """
    Delete metadata from store and cache
    """
    query_key = (session_id, chunk_key)
    try:
        del self._meta_store[query_key]
    except KeyError:
        pass
    try:
        del self._meta_cache[query_key]
    except KeyError:
        pass

    # broadcast deletion into pre-determined destinations
    futures = []
    if query_key in self._meta_broadcasts:
        for dest in self._meta_broadcasts[query_key]:
            futures.append(
                self.ctx.actor_ref(self.default_name(), address=dest).delete_meta(
                    session_id, chunk_key, _wait=False, _tell=True
                )
            )
    if self._kv_store_ref is not None:
        futures.append(
            self._kv_store_ref.delete(
                "/sessions/%s/chunks/%s" % (session_id, chunk_key),
                recursive=True,
                _tell=True,
                _wait=False,
            )
        )
    [f.result() for f in futures]


# Traceback (KeyError)
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 716, in gevent._greenlet.Greenlet.run
File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
cpdef object fire_run(self):
File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
with self.lock:
File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
raise
File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
res = actor.on_receive(message_ctx.message)
File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
cpdef on_receive(self, message):
File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
return getattr(self, method)(*args, **kwargs)
File "/home/admin/wenjun.swj/mars/mars/scheduler/kvstore.py", line 54, in delete
return self._store.delete(key, dir=dir, recursive=recursive)
File "/home/admin/wenjun.swj/mars/mars/kvstore.py", line 265, in delete
raise KeyError(key)
KeyError: '/sessions/c3c9d252-09bc-11e9-bae9-97dedca03eb9/chunks/6f02ffa9ca5cf0b82362b37418d29ccc'
2018-12-27T10:17:46Z <Greenlet "Greenlet-0" at 0x7fed5e0f6268: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0x7fed5f8bbf48>> failed with KeyError

# Fixed code
def delete_meta(self, session_id, chunk_key):
    """
    Delete metadata from store and cache
    """
    query_key = (session_id, chunk_key)
    try:
        del self._meta_store[query_key]
        if self._kv_store_ref is not None:
            self._kv_store_ref.delete(
                "/sessions/%s/chunks/%s" % (session_id, chunk_key),
                recursive=True,
                _tell=True,
                _wait=False,
            )
    except KeyError:
        pass
    try:
        del self._meta_cache[query_key]
    except KeyError:
        pass

    # broadcast deletion into pre-determined destinations
    if query_key in self._meta_broadcasts:
        for dest in self._meta_broadcasts[query_key]:
            self.ctx.actor_ref(self.default_name(), address=dest).delete_meta(
                session_id, chunk_key, _wait=False, _tell=True
            )