# Buggy code
def _execute_operand(self, op):
    results = self._chunk_results
    ref_counts = self._chunk_key_ref_counts
    op_keys = self._executed_op_keys
    try:
        ops = list(self._op_key_to_ops[op.key])
        if not self._mock:
            # do real execution
            # note that currently execution is the chunk-level
            # so we pass the first operand's first output to Executor.handle
            first_op = ops[0]
            Executor.handle(first_op.outputs[0], results)
            op_keys.add(first_op.key)
            # handle other operands
            for rest_op in ops[1:]:
                for op_output, rest_op_output in zip(first_op.outputs, rest_op.outputs):
                    results[rest_op_output.key] = results[op_output.key]
        else:
            sparse_percent = self._sparse_mock_percent if op.sparse else 1.0
            for output in op.outputs:
                results[output.key] = output.nbytes * sparse_percent

        with self._lock:
            for output in itertools.chain(*[op.outputs for op in ops]):
                # the output not in the graph will be skipped
                if output not in self._graph:
                    continue
                # in case that operand has multiple outputs
                # and some of the output not in result keys, delete them
                if ref_counts.get(output.key) == 0:
                    del results[output.key]

                # clean the predecessors' results if ref counts equals 0
                for pred_chunk in self._graph.iter_predecessors(output):
                    if pred_chunk.key in ref_counts:
                        ref_counts[pred_chunk.key] -= 1
                        if ref_counts[pred_chunk.key] == 0:
                            del results[pred_chunk.key]

                # add successors' operands to queue
                for succ_chunk in self._graph.iter_successors(output):
                    preds = self._graph.predecessors(succ_chunk)
                    if succ_chunk.op.key not in self._submitted_op_keys and (
                        len(preds) == 0 or all(pred.op.key in op_keys for pred in preds)
                    ):
                        self._queue.insert(0, succ_chunk.op)
    except Exception:
        self._has_error.set()
        raise
    finally:
        self._semaphore.release()


# Traceback (KeyError)
In [1]: import mars.tensor as mt

In [2]: a = mt.ones((10, 5), chunk_size=5)

In [3]: s = mt.linalg.svd(a)[1]

In [4]: s.execute()
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-4-4a5966ced510> in <module>
----> 1 s.execute()

~/Documents/mars_dev/mars/mars/tensor/core.py in execute(self, session, **kw)
378
379     def execute(self, session=None, **kw):
--> 380         return self._data.execute(session, **kw)
381
382

~/Documents/mars_dev/mars/mars/tensor/core.py in execute(self, session, **kw)
184         if session is None:
185             session = Session.default_or_local()
--> 186         return session.run(self, **kw)
187
188     def fetch(self, session=None, **kw):

~/Documents/mars_dev/mars/mars/session.py in run(self, *tensors, **kw)
107
108         tensors = tuple(mt.tensor(t) for t in tensors)
--> 109         result = self._sess.run(*tensors, **kw)
110
111         for t in tensors:

~/Documents/mars_dev/mars/mars/session.py in run(self, *tensors, **kw)
49         if 'n_parallel' not in kw:
50             kw['n_parallel'] = cpu_count()
---> 51         res = self._executor.execute_tensors(tensors, **kw)
52         return res
53

~/Documents/mars_dev/mars/mars/utils.py in _wrapped(*args, **kwargs)
343     def _wrapped(*args, **kwargs):
344         _kernel_mode.eager = False
--> 345         return_value = func(*args, **kwargs)
346         _kernel_mode.eager = None
347         return return_value

~/Documents/mars_dev/mars/mars/executor.py in execute_tensors(self, tensors, fetch, n_parallel, n_thread, print_progress, mock, sparse_mock_percent)
473         self.execute_graph(graph, result_keys, n_parallel=n_parallel or n_thread,
474                            print_progress=print_progress, mock=mock,
--> 475                            sparse_mock_percent=sparse_mock_percent)
476
477         results = self._chunk_result

~/Documents/mars_dev/mars/mars/executor.py in execute_graph(self, graph, keys, n_parallel, print_progress, mock, sparse_mock_percent)
410                                          print_progress=print_progress, mock=mock,
411                                          sparse_mock_percent=sparse_mock_percent)
--> 412         return graph_execution.execute(True)
413
414     @kernel_mode

~/Documents/mars_dev/mars/mars/executor.py in execute(self, retval)
339         # wait until all the futures completed
340         for future in executed_futures:
--> 341             future.result()
342
343         if self._mock:

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
423                 raise CancelledError()
424             elif self._state == FINISHED:
--> 425                 return self.__get_result()
426
427             self._condition.wait(timeout)

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
382     def __get_result(self):
383         if self._exception:
--> 384             raise self._exception
385         else:
386             return self._result

~/miniconda3/lib/python3.7/concurrent/futures/thread.py in run(self)
55
56         try:
---> 57             result = self.fn(*self.args, **self.kwargs)
58         except BaseException as exc:
59             self.future.set_exception(exc)

~/Documents/mars_dev/mars/mars/executor.py in _execute_operand(self, op)
246                     # and some of the output not in result keys, delete them
247                     if ref_counts.get(output.key) == 0:
--> 248                         del results[output.key]
249
250                     # clean the predecessors' results if ref counts equals 0

KeyError: '4ec5b76155f38976c84e36d156349503'

# Fixed code
def _execute_operand(self, op):
    results = self._chunk_results
    ref_counts = self._chunk_key_ref_counts
    op_keys = self._executed_op_keys
    executed_chunk_keys = set()
    deleted_chunk_keys = set()
    try:
        ops = list(self._op_key_to_ops[op.key])
        if not self._mock:
            # do real execution
            # note that currently execution is the chunk-level
            # so we pass the first operand's first output to Executor.handle
            first_op = ops[0]
            Executor.handle(first_op.outputs[0], results)
            executed_chunk_keys.update([c.key for c in first_op.outputs])
            op_keys.add(first_op.key)
            # handle other operands
            for rest_op in ops[1:]:
                for op_output, rest_op_output in zip(first_op.outputs, rest_op.outputs):
                    # if the op's outputs have been stored,
                    # other same key ops' results will be the same
                    if rest_op_output.key not in executed_chunk_keys:
                        results[rest_op_output.key] = results[op_output.key]
        else:
            sparse_percent = self._sparse_mock_percent if op.sparse else 1.0
            for output in op.outputs:
                results[output.key] = output.nbytes * sparse_percent

        with self._lock:
            for output in itertools.chain(*[op.outputs for op in ops]):
                # the output not in the graph will be skipped
                if output not in self._graph:
                    continue
                # in case that operand has multiple outputs
                # and some of the output not in result keys, delete them
                if ref_counts.get(output.key) == 0:
                    # if the result has been deleted, it should be skipped
                    if output.key not in deleted_chunk_keys:
                        deleted_chunk_keys.add(output.key)
                        del results[output.key]

                # clean the predecessors' results if ref counts equals 0
                for pred_chunk in self._graph.iter_predecessors(output):
                    if pred_chunk.key in ref_counts:
                        ref_counts[pred_chunk.key] -= 1
                        if ref_counts[pred_chunk.key] == 0:
                            del results[pred_chunk.key]

                # add successors' operands to queue
                for succ_chunk in self._graph.iter_successors(output):
                    preds = self._graph.predecessors(succ_chunk)
                    if succ_chunk.op.key not in self._submitted_op_keys and (
                        len(preds) == 0 or all(pred.op.key in op_keys for pred in preds)
                    ):
                        self._queue.insert(0, succ_chunk.op)
    except Exception:
        self._has_error.set()
        raise
    finally:
        self._semaphore.release()