# Buggy code
def stop(self):
    try:
        for actor in (
            self._cpu_calc_actors
            + self._sender_actors
            + self._inproc_holder_actors
            + self._inproc_io_runner_actors
            + self._cuda_calc_actors
            + self._cuda_holder_actors
            + self._receiver_actors
            + self._spill_actors
            + self._process_helper_actors
        ):
            if actor and actor.ctx:
                actor.destroy(wait=False)

        if self._result_sender_ref:
            self._result_sender_ref.destroy(wait=False)
        if self._status_ref:
            self._status_ref.destroy(wait=False)
        if self._shared_holder_ref:
            self._shared_holder_ref.destroy(wait=False)
        if self._storage_manager_ref:
            self._storage_manager_ref.destroy(wait=False)
        if self._events_ref:
            self._events_ref.destroy(wait=False)
        if self._dispatch_ref:
            self._dispatch_ref.destroy(wait=False)
        if self._execution_ref:
            self._execution_ref.destroy(wait=False)
    finally:
        self._plasma_store.__exit__(None, None, None)


# Traceback (KeyError)
In [1]: import mars.remote as mr

In [2]: from mars.deploy.local import new_cluster

In [3]: c = new_cluster()
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0604 14:30:01.529353 132840896 store.cc:1149] Allowing the Plasma store to use up to 3.43597GB of memory.
I0604 14:30:01.534931 132840896 store.cc:1176] Starting object store with directory /tmp and huge page support disabled

In [4]: def f(x):
...:     return
...:

In [5]: mr.spawn(f, 3).execute()
Unhandled exception in promise
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unhandled exception in promise
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unexpected error occurred in executing graph e0879bcab1971118787b182578bff091
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Attempt 1: Unexpected error KeyError occurred in executing operand e0879bcab1971118787b182578bff091 in 0.0.0.0:37928
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unhandled exception in promise
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unhandled exception in promise
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unexpected error occurred in executing graph e0879bcab1971118787b182578bff091
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Attempt 2: Unexpected error KeyError occurred in executing operand e0879bcab1971118787b182578bff091 in 0.0.0.0:37928
Traceback (most recent call last):
File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'

# Fixed code
def stop(self):
    try:
        destroy_futures = []
        for actor in (
            self._cpu_calc_actors
            + self._sender_actors
            + self._inproc_holder_actors
            + self._inproc_io_runner_actors
            + self._cuda_calc_actors
            + self._cuda_holder_actors
            + self._receiver_actors
            + self._spill_actors
            + self._process_helper_actors
        ):
            if actor and actor.ctx:
                destroy_futures.append(actor.destroy(wait=False))

        if self._result_sender_ref:
            destroy_futures.append(self._result_sender_ref.destroy(wait=False))
        if self._status_ref:
            destroy_futures.append(self._status_ref.destroy(wait=False))
        if self._shared_holder_ref:
            destroy_futures.append(self._shared_holder_ref.destroy(wait=False))
        if self._storage_manager_ref:
            destroy_futures.append(self._storage_manager_ref.destroy(wait=False))
        if self._events_ref:
            destroy_futures.append(self._events_ref.destroy(wait=False))
        if self._dispatch_ref:
            destroy_futures.append(self._dispatch_ref.destroy(wait=False))
        if self._execution_ref:
            destroy_futures.append(self._execution_ref.destroy(wait=False))
        [f.result(5) for f in destroy_futures]
    finally:
        self._plasma_store.__exit__(None, None, None)