# Buggy code
def _handle_worker_accept(self, worker):
    def _dequeue_worker(endpoint, wait=True):
        try:
            with rewrite_worker_errors():
                return self._get_execution_ref(address=endpoint).dequeue_graph(
                    self._session_id, self._op_key, _tell=True, _wait=wait
                )
        finally:
            self._assigned_workers.difference_update((worker,))

    if self._position == OperandPosition.INITIAL:
        new_worker = self._graph_ref.get_operand_target_worker(self._op_key)
        if new_worker and new_worker != self._target_worker:
            logger.debug(
                "Cancelling running operand %s on %s, new_target %s",
                self._op_key,
                worker,
                new_worker,
            )
            _dequeue_worker(worker)
            return

    if (self.worker and self.worker != worker) or (
        self._target_worker and worker != self._target_worker
    ):
        logger.debug(
            "Cancelling running operand %s on %s, op_worker %s, op_target %s",
            self._op_key,
            worker,
            self.worker,
            self._target_worker,
        )
        _dequeue_worker(worker)
        return
    elif self.worker is not None:
        logger.debug("Worker for operand %s already assigned", self._op_key)
        return

    # worker assigned, submit job
    if self.state in (OperandState.CANCELLED, OperandState.CANCELLING):
        self.ref().start_operand(_tell=True)
        return

    if worker != self.worker:
        self._execution_ref = None
    self.worker = worker
    cancel_futures = []
    for w in list(self._assigned_workers):
        if w != worker:
            logger.debug(
                "Cancelling running operand %s on %s, when deciding to run on %s",
                self._op_key,
                w,
                worker,
            )
            cancel_futures.append(_dequeue_worker(w, wait=False))

    for f in cancel_futures:
        with rewrite_worker_errors(ignore_error=True):
            f.result()
    self._assigned_workers = set()

    target_predicts = self._get_target_predicts(worker)

    # prepare meta broadcasts
    broadcast_eps = set()
    for succ_key in self._succ_keys:
        broadcast_eps.add(self.get_scheduler(self.gen_uid(self._session_id, succ_key)))
    broadcast_eps.difference_update({self.address})
    broadcast_eps = tuple(broadcast_eps)

    chunk_keys, broadcast_ep_groups = [], []
    for chunk_key in self._chunks:
        chunk_keys.append(chunk_key)
        broadcast_ep_groups.append(broadcast_eps)

    self._chunk_meta_ref.batch_set_chunk_broadcasts(
        self._session_id, chunk_keys, broadcast_ep_groups, _tell=True, _wait=False
    )

    # submit job
    logger.debug("Start running operand %s on %s", self._op_key, worker)
    self._execution_ref = self._get_execution_ref()
    try:
        with rewrite_worker_errors():
            self._execution_ref.start_execution(
                self._session_id,
                self._op_key,
                send_addresses=target_predicts,
                _promise=True,
            )
    except WorkerDead:
        self._resource_ref.detach_dead_workers([self.worker], _tell=True)
        return
    # here we start running immediately to avoid accidental state change
    # and potential submission
    self.start_operand(OperandState.RUNNING)


# Traceback (SystemError)
In [1]: from mars.deploy.local import new_cluster

In [2]: import mars.tensor as mt

In [3]: cluster = new_cluster(scheduler_n_process=2, worker_n_process=2)
/Users/travis/miniconda3/conda-bld/arrow-cpp_1540410566532/work/cpp/src/plasma/store.cc:971: Allowing the Plasma store to use up to 3.43597GB of memory.
/Users/travis/miniconda3/conda-bld/arrow-cpp_1540410566532/work/cpp/src/plasma/store.cc:1001: Starting object store with directory /tmp and huge page support disabled
/Users/travis/miniconda3/conda-bld/arrow-cpp_1540410566532/work/cpp/src/plasma/eviction_policy.cc:85: There is not enough space to create this object, so evicting 0 objects to free up 0 bytes. The number of bytes in use (before this eviction) is 2130706432.

In [4]: a = mt.ones((3, 3), chunk_size=2)

In [5]: cluster.session.run(a)
Out[5]:
array([[1., 1., 1.],
[1., 1., 1.],
[1., 1., 1.]])

In [6]: cluster.session.run(a.dot(a))
Failed to start graph execution.
Traceback (most recent call last):
File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 249, in execute_graph
self.create_operand_actors()
File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 679, in create_operand_actors
op_refs = dict((k, v.result()) for k, v in op_refs.items())
File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 679, in <genexpr>
op_refs = dict((k, v.result()) for k, v in op_refs.items())
File "src/gevent/event.py", line 457, in gevent._event.AsyncResult.result
File "src/gevent/event.py", line 381, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 409, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 399, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 379, in gevent._event.AsyncResult._raise_exception
File "/Users/hekaisheng/miniconda3/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise
raise value.with_traceback(tb)
File "mars/actors/pool/gevent_pool.pyx", line 984, in mars.actors.pool.gevent_pool.Communicator._on_receive_create_actor
self.pool.create_actor(message.actor_cls, message.actor_ref.uid,
File "mars/actors/pool/gevent_pool.pyx", line 241, in mars.actors.pool.gevent_pool.LocalActorPool.create_actor
raise ActorAlreadyExist('Actor {0} already exist, cannot create'.format(actor.uid))
mars.actors.errors.ActorAlreadyExist: Actor s:operator$8a307641-78a8-43a9-a612-8edca02ca309$058707656a679b3a0ab03c10d40abb0e already exist, cannot create
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 716, in gevent._greenlet.Greenlet.run
File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
cpdef object fire_run(self):
File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
with self.lock:
File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
raise
File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
res = actor.on_receive(message_ctx.message)
File "mars/actors/core.pyx", line 108, in mars.actors.core.FunctionActor.on_receive
File "mars/actors/core.pyx", line 110, in mars.actors.core.FunctionActor.on_receive
File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 249, in execute_graph
self.create_operand_actors()
File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 679, in create_operand_actors
op_refs = dict((k, v.result()) for k, v in op_refs.items())
File "/Users/hekaisheng/Documents/mars_dev/mars/mars/scheduler/graph.py", line 679, in <genexpr>
op_refs = dict((k, v.result()) for k, v in op_refs.items())
File "src/gevent/event.py", line 457, in gevent._event.AsyncResult.result
File "src/gevent/event.py", line 381, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 409, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 399, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 379, in gevent._event.AsyncResult._raise_exception
File "/Users/hekaisheng/miniconda3/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise
raise value.with_traceback(tb)
File "mars/actors/pool/gevent_pool.pyx", line 984, in mars.actors.pool.gevent_pool.Communicator._on_receive_create_actor
self.pool.create_actor(message.actor_cls, message.actor_ref.uid,
File "mars/actors/pool/gevent_pool.pyx", line 241, in mars.actors.pool.gevent_pool.LocalActorPool.create_actor
raise ActorAlreadyExist('Actor {0} already exist, cannot create'.format(actor.uid))
mars.actors.errors.ActorAlreadyExist: Actor s:operator$8a307641-78a8-43a9-a612-8edca02ca309$058707656a679b3a0ab03c10d40abb0e already exist, cannot create
2019-01-24T12:05:09Z <Greenlet "Greenlet-0" at 0x119218448: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0x319839688>> failed with ActorAlreadyExist

---------------------------------------------------------------------------
SystemError                               Traceback (most recent call last)
<ipython-input-6-d3363e4685e3> in <module>()
----> 1 cluster.session.run(a.dot(a))

~/Documents/mars_dev/mars/mars/session.py in run(self, *tensors, **kw)
115         # execute the non-executed tensors
116         if idx_to_run_tensors:
--> 117             execute_result = self._sess.run(*idx_to_run_tensors.values(), **kw)
118             if execute_result:
119                 # fetch is True

~/Documents/mars_dev/mars/mars/deploy/local/session.py in run(self, *tensors, **kw)
80             if graph_state == GraphState.FAILED:
81                 # TODO(qin): add traceback
---> 82                 raise SystemError('Graph execution failed with unknown reason')
83
84         if 0 < timeout < time.time() - exec_start_time:

SystemError: Graph execution failed with unknown reason

# Fixed code
def _handle_worker_accept(self, worker):
    def _dequeue_worker(endpoint, wait=True):
        try:
            with rewrite_worker_errors():
                return self._get_execution_ref(address=endpoint).dequeue_graph(
                    self._session_id, self._op_key, _tell=True, _wait=wait
                )
        finally:
            self._assigned_workers.difference_update((worker,))

    if self._position == OperandPosition.INITIAL:
        new_worker = self._graph_refs[0].get_operand_target_worker(self._op_key)
        if new_worker and new_worker != self._target_worker:
            logger.debug(
                "Cancelling running operand %s on %s, new_target %s",
                self._op_key,
                worker,
                new_worker,
            )
            _dequeue_worker(worker)
            return

    if (self.worker and self.worker != worker) or (
        self._target_worker and worker != self._target_worker
    ):
        logger.debug(
            "Cancelling running operand %s on %s, op_worker %s, op_target %s",
            self._op_key,
            worker,
            self.worker,
            self._target_worker,
        )
        _dequeue_worker(worker)
        return
    elif self.worker is not None:
        logger.debug("Worker for operand %s already assigned", self._op_key)
        return

    # worker assigned, submit job
    if self.state in (OperandState.CANCELLED, OperandState.CANCELLING):
        self.ref().start_operand(_tell=True)
        return

    if worker != self.worker:
        self._execution_ref = None
    self.worker = worker
    cancel_futures = []
    for w in list(self._assigned_workers):
        if w != worker:
            logger.debug(
                "Cancelling running operand %s on %s, when deciding to run on %s",
                self._op_key,
                w,
                worker,
            )
            cancel_futures.append(_dequeue_worker(w, wait=False))

    for f in cancel_futures:
        with rewrite_worker_errors(ignore_error=True):
            f.result()
    self._assigned_workers = set()

    target_predicts = self._get_target_predicts(worker)

    # prepare meta broadcasts
    broadcast_eps = set()
    for succ_key in self._succ_keys:
        broadcast_eps.add(self.get_scheduler(self.gen_uid(self._session_id, succ_key)))
    broadcast_eps.difference_update({self.address})
    broadcast_eps = tuple(broadcast_eps)

    chunk_keys, broadcast_ep_groups = [], []
    for chunk_key in self._chunks:
        chunk_keys.append(chunk_key)
        broadcast_ep_groups.append(broadcast_eps)

    self._chunk_meta_ref.batch_set_chunk_broadcasts(
        self._session_id, chunk_keys, broadcast_ep_groups, _tell=True, _wait=False
    )

    # submit job
    logger.debug("Start running operand %s on %s", self._op_key, worker)
    self._execution_ref = self._get_execution_ref()
    try:
        with rewrite_worker_errors():
            self._execution_ref.start_execution(
                self._session_id,
                self._op_key,
                send_addresses=target_predicts,
                _promise=True,
            )
    except WorkerDead:
        self._resource_ref.detach_dead_workers([self.worker], _tell=True)
        return
    # here we start running immediately to avoid accidental state change
    # and potential submission
    self.start_operand(OperandState.RUNNING)