# Buggy code
def prepare_graph(self, compose=True):
    """
    Tile and compose tensor graph into chunk graph
    :param compose: if True, do compose after tiling
    """
    tileable_graph = deserialize_graph(self._serialized_tileable_graph)
    self._tileable_graph_cache = tileable_graph

    logger.debug(
        "Begin preparing graph %s with %d tensors to chunk graph.",
        self._graph_key,
        len(tileable_graph),
    )

    # mark target tensor steps
    if not self._target_tileable_chunk_ops:
        for tn in tileable_graph:
            if not tileable_graph.count_successors(tn):
                self._target_tileable_chunk_ops[tn.key] = set()
                self._target_tileable_finished[tn.key] = set()

    if self._serialized_chunk_graph:
        serialized_chunk_graph = self._serialized_chunk_graph
        chunk_graph = DAG.from_pb(serialized_chunk_graph)
    else:
        chunk_graph = DAG()

    key_to_chunk = {c.key: c for c in chunk_graph}

    tileable_key_opid_to_tiled = self._tileable_key_opid_to_tiled

    for t in tileable_graph:
        self._tileable_key_to_opid[t.key] = t.op.id
        if (t.key, t.op.id) not in tileable_key_opid_to_tiled:
            continue
        t._chunks = [
            key_to_chunk[k] for k in [tileable_key_opid_to_tiled[(t.key, t.op.id)][-1]]
        ]

    tq = deque()
    for t in tileable_graph:
        if t.inputs and not all(
            (ti.key, ti.op.id) in tileable_key_opid_to_tiled for ti in t.inputs
        ):
            continue
        tq.append(t)

    while tq:
        tileable = tq.popleft()
        if (
            not tileable.is_coarse()
            or (tileable.key, tileable.op.id) in tileable_key_opid_to_tiled
        ):
            continue
        inputs = [
            tileable_key_opid_to_tiled[(it.key, it.op.id)][-1]
            for it in tileable.inputs or ()
        ]

        op = tileable.op.copy()
        _ = op.new_tileables(
            inputs,  # noqa: F841
            kws=[o.params for o in tileable.op.outputs],
            output_limit=len(tileable.op.outputs),
            **tileable.extra_params,
        )

        total_tiled = []
        for j, t, to_tile in zip(itertools.count(0), tileable.op.outputs, op.outputs):
            # replace inputs with tiled ones
            if not total_tiled:
                try:
                    if isinstance(to_tile.op, Fetch):
                        td = self.tile_fetch_tileable(tileable)
                    else:
                        td = self._graph_analyze_pool.submit(
                            handler.dispatch, to_tile
                        ).result()
                except DataNotReady:
                    continue

                if isinstance(td, (tuple, list)):
                    total_tiled.extend(td)
                else:
                    total_tiled.append(td)

            tiled = total_tiled[j]
            tileable_key_opid_to_tiled[(t.key, t.op.id)].append(tiled)

            # add chunks to fine grained graph
            q = deque(
                [
                    tiled_c if isinstance(tiled_c, ChunkData) else tiled_c.data
                    for tiled_c in tiled.chunks
                ]
            )
            input_chunk_keys = set(
                itertools.chain(
                    *(
                        [(it.key, it.id) for it in input.chunks]
                        for input in to_tile.inputs
                    )
                )
            )
            while len(q) > 0:
                c = q.popleft()
                if (c.key, c.id) in input_chunk_keys:
                    continue
                if c not in chunk_graph:
                    chunk_graph.add_node(c)
                for ic in c.inputs or []:
                    if ic not in chunk_graph:
                        chunk_graph.add_node(ic)
                        q.append(ic)
                    chunk_graph.add_edge(ic, c)

            for succ in tileable_graph.successors(t):
                if any(
                    (t.key, t.op.id) not in tileable_key_opid_to_tiled
                    for t in succ.inputs
                ):
                    continue
                tq.append(succ)

    # record the chunk nodes in graph
    reserve_chunk = set()
    result_chunk_keys = list()
    for tk, topid in tileable_key_opid_to_tiled:
        if tk not in self._target_tileable_chunk_ops:
            continue
        for n in [
            c.data for t in tileable_key_opid_to_tiled[(tk, topid)] for c in t.chunks
        ]:
            result_chunk_keys.append(n.key)
            dq_predecessors = deque([n])
            while dq_predecessors:
                current = dq_predecessors.popleft()
                reserve_chunk.update(n.op.outputs)
                predecessors = chunk_graph.predecessors(current)
                dq_predecessors.extend(
                    [p for p in predecessors if p not in reserve_chunk]
                )
                reserve_chunk.update(predecessors)
    # delete redundant chunk
    for n in list(chunk_graph.iter_nodes()):
        if n not in reserve_chunk:
            chunk_graph.remove_node(n)
        elif isinstance(n.op, Fetch):
            chunk_graph.remove_node(n)

    if compose:
        chunk_graph.compose(keys=result_chunk_keys)

    for tk, topid in tileable_key_opid_to_tiled:
        if tk not in self._target_tileable_chunk_ops:
            continue
        for n in tileable_key_opid_to_tiled[(tk, topid)][-1].chunks:
            self._terminal_chunk_op_tileable[n.op.key].add(tk)
            self._target_tileable_chunk_ops[tk].add(n.op.key)

    # sync chunk graph to kv store
    if self._kv_store_ref is not None:
        graph_path = "/sessions/%s/graphs/%s" % (self._session_id, self._graph_key)
        self._kv_store_ref.write(
            "%s/chunk_graph" % graph_path,
            serialize_graph(chunk_graph, compress=True),
            _tell=True,
            _wait=False,
        )

    self._chunk_graph_cache = chunk_graph
    for n in self._chunk_graph_cache:
        self._op_key_to_chunk[n.op.key].append(n)


# Traceback (TypeError)
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "mars/actors/pool/gevent_pool.pyx", line 88, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
File "mars/actors/pool/gevent_pool.pyx", line 91, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
File "mars/actors/pool/gevent_pool.pyx", line 102, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
File "mars/actors/core.pyx", line 112, in mars.actors.core._FunctionActor.on_receive
File "mars/actors/core.pyx", line 114, in mars.actors.core._FunctionActor.on_receive
File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 288, in _wrapped
return func(*args, **kwargs)
File "/Users/wenjun.swj/Code/mars/mars/scheduler/graph.py", line 335, in execute_graph
self.prepare_graph(compose=compose)
File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 288, in _wrapped
return func(*args, **kwargs)
File "/Users/wenjun.swj/Code/mars/mars/scheduler/graph.py", line 475, in prepare_graph
td = self.tile_fetch_tileable(tileable)
File "/Users/wenjun.swj/Code/mars/mars/scheduler/graph.py", line 936, in tile_fetch_tileable
fetch_graph = deserialize_graph(graph_ref.build_fetch_graph(tileable_key))
File "mars/actors/core.pyx", line 65, in mars.actors.core.ActorRef.__getattr__._mt_call
File "mars/actors/core.pyx", line 37, in mars.actors.core.ActorRef.send
File "mars/actors/pool/gevent_pool.pyx", line 179, in mars.actors.pool.gevent_pool.ActorContext.send
File "mars/actors/pool/gevent_pool.pyx", line 754, in mars.actors.pool.gevent_pool.Communicator.send
File "mars/actors/pool/gevent_pool.pyx", line 755, in mars.actors.pool.gevent_pool.Communicator.send
File "mars/actors/pool/gevent_pool.pyx", line 750, in mars.actors.pool.gevent_pool.Communicator._send
File "mars/actors/pool/gevent_pool.pyx", line 657, in mars.actors.pool.gevent_pool.Communicator._dispatch
File "mars/actors/pool/gevent_pool.pyx", line 682, in mars.actors.pool.gevent_pool.Communicator._send_local
File "src/gevent/greenlet.py", line 709, in gevent._greenlet.Greenlet.get
File "src/gevent/greenlet.py", line 317, in gevent._greenlet.Greenlet._raise_exception
File "/Users/wenjun.swj/miniconda3/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
raise value.with_traceback(tb)
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "mars/actors/pool/gevent_pool.pyx", line 72, in mars.actors.pool.gevent_pool.MessageContext.result
File "mars/actors/pool/gevent_pool.pyx", line 73, in mars.actors.pool.gevent_pool.MessageContext.result
File "src/gevent/event.py", line 344, in gevent._event.AsyncResult.result
File "src/gevent/event.py", line 268, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 286, in gevent._event.AsyncResult.get
File "src/gevent/event.py", line 266, in gevent._event.AsyncResult._raise_exception
File "/Users/wenjun.swj/miniconda3/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
raise value.with_traceback(tb)
File "mars/actors/pool/gevent_pool.pyx", line 96, in mars.actors.pool.gevent_pool.ActorExecutionContext.fire_run
File "mars/actors/core.pyx", line 112, in mars.actors.core._FunctionActor.on_receive
File "mars/actors/core.pyx", line 114, in mars.actors.core._FunctionActor.on_receive
File "/Users/wenjun.swj/Code/mars/mars/scheduler/graph.py", line 928, in build_fetch_graph
return serialize_graph(graph)
File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 217, in serialize_graph
ser_graph = graph.to_pb().SerializeToString()
File "mars/graph.pyx", line 431, in mars.graph.DirectedGraph.to_pb
File "mars/serialize/core.pyx", line 563, in mars.serialize.core.Serializable.to_pb
File "mars/serialize/core.pyx", line 547, in mars.serialize.core.Serializable.serialize
File "mars/serialize/core.pyx", line 665, in mars.serialize.core.Provider.serialize_model
File "mars/serialize/core.pyx", line 141, in mars.serialize.core.Field.serialize
File "mars/serialize/core.pyx", line 142, in mars.serialize.core.Field.serialize
File "mars/serialize/pbserializer.pyx", line 476, in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field
File "mars/serialize/core.pyx", line 547, in mars.serialize.core.Serializable.serialize
File "mars/serialize/core.pyx", line 665, in mars.serialize.core.Provider.serialize_model
File "mars/serialize/core.pyx", line 141, in mars.serialize.core.Field.serialize
File "mars/serialize/core.pyx", line 142, in mars.serialize.core.Field.serialize
File "mars/serialize/pbserializer.pyx", line 447, in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field
File "mars/serialize/core.pyx", line 142, in mars.serialize.core.Field.serialize
File "mars/serialize/pbserializer.pyx", line 467, in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field
File "mars/serialize/core.pyx", line 547, in mars.serialize.core.Serializable.serialize
File "mars/serialize/core.pyx", line 665, in mars.serialize.core.Provider.serialize_model
File "mars/serialize/core.pyx", line 141, in mars.serialize.core.Field.serialize
File "mars/serialize/core.pyx", line 142, in mars.serialize.core.Field.serialize
File "mars/serialize/pbserializer.pyx", line 485, in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field
File "mars/serialize/pbserializer.pyx", line 428, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_value
File "mars/serialize/pbserializer.pyx", line 365, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_typed_value
File "mars/serialize/pbserializer.pyx", line 246, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_tuple
File "mars/serialize/pbserializer.pyx", line 428, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_value
File "mars/serialize/pbserializer.pyx", line 365, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_typed_value
File "mars/serialize/pbserializer.pyx", line 246, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_tuple
File "mars/serialize/pbserializer.pyx", line 428, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_value
File "mars/serialize/pbserializer.pyx", line 338, in mars.serialize.pbserializer.ProtobufSerializeProvider._set_typed_value
TypeError: nan has type float, but expected one of: int, long
2019-05-21T10:36:49Z <Greenlet at 0xc20a0d598: <built-in method fire_run of mars.actors.pool.gevent_pool.ActorExecutionContext object at 0xc20d32138>> failed with TypeError

# Fixed code
def prepare_graph(self, compose=True):
    """
    Tile and compose tileable graph into chunk graph
    :param compose: if True, do compose after tiling
    """
    tileable_graph = deserialize_graph(self._serialized_tileable_graph)
    self._tileable_graph_cache = tileable_graph

    logger.debug(
        "Begin preparing graph %s with %d tileables to chunk graph.",
        self._graph_key,
        len(tileable_graph),
    )

    if not self._target_tileable_chunk_ops:
        for tn in tileable_graph:
            if not tileable_graph.count_successors(tn):
                self._target_tileable_chunk_ops[tn.key] = set()
                self._target_tileable_finished[tn.key] = set()

    if self._serialized_chunk_graph:
        serialized_chunk_graph = self._serialized_chunk_graph
        chunk_graph = DAG.from_pb(serialized_chunk_graph)
    else:
        chunk_graph = DAG()

    key_to_chunk = {c.key: c for c in chunk_graph}

    tileable_key_opid_to_tiled = self._tileable_key_opid_to_tiled

    for t in tileable_graph:
        self._tileable_key_to_opid[t.key] = t.op.id
        if (t.key, t.op.id) not in tileable_key_opid_to_tiled:
            continue
        t._chunks = [
            key_to_chunk[k] for k in [tileable_key_opid_to_tiled[(t.key, t.op.id)][-1]]
        ]

    tq = deque()
    for t in tileable_graph:
        if t.inputs and not all(
            (ti.key, ti.op.id) in tileable_key_opid_to_tiled for ti in t.inputs
        ):
            continue
        tq.append(t)

    while tq:
        tileable = tq.popleft()
        if (
            not tileable.is_coarse()
            or (tileable.key, tileable.op.id) in tileable_key_opid_to_tiled
        ):
            continue
        inputs = [
            tileable_key_opid_to_tiled[(it.key, it.op.id)][-1]
            for it in tileable.inputs or ()
        ]

        op = tileable.op.copy()
        _ = op.new_tileables(
            inputs,  # noqa: F841
            kws=[o.params for o in tileable.op.outputs],
            output_limit=len(tileable.op.outputs),
            **tileable.extra_params,
        )

        total_tiled = []
        for j, t, to_tile in zip(itertools.count(0), tileable.op.outputs, op.outputs):
            # replace inputs with tiled ones
            if not total_tiled:
                try:
                    if isinstance(to_tile.op, Fetch):
                        td = self.tile_fetch_tileable(tileable)
                    else:
                        td = self._graph_analyze_pool.submit(
                            handler.dispatch, to_tile
                        ).result()
                except DataNotReady:
                    continue

                if isinstance(td, (tuple, list)):
                    total_tiled.extend(td)
                else:
                    total_tiled.append(td)

            tiled = total_tiled[j]
            tileable_key_opid_to_tiled[(t.key, t.op.id)].append(tiled)

            # add chunks to fine grained graph
            q = deque(
                [
                    tiled_c if isinstance(tiled_c, ChunkData) else tiled_c.data
                    for tiled_c in tiled.chunks
                ]
            )
            input_chunk_keys = set(
                itertools.chain(
                    *(
                        [(it.key, it.id) for it in input.chunks]
                        for input in to_tile.inputs
                    )
                )
            )
            while len(q) > 0:
                c = q.popleft()
                if (c.key, c.id) in input_chunk_keys:
                    continue
                if c not in chunk_graph:
                    chunk_graph.add_node(c)
                for ic in c.inputs or []:
                    if ic not in chunk_graph:
                        chunk_graph.add_node(ic)
                        q.append(ic)
                    chunk_graph.add_edge(ic, c)

            for succ in tileable_graph.successors(t):
                if any(
                    (t.key, t.op.id) not in tileable_key_opid_to_tiled
                    for t in succ.inputs
                ):
                    continue
                tq.append(succ)

    # record the chunk nodes in graph
    reserve_chunk = set()
    result_chunk_keys = list()
    for tk, topid in tileable_key_opid_to_tiled:
        if tk not in self._target_tileable_chunk_ops:
            continue
        for n in [
            c.data for t in tileable_key_opid_to_tiled[(tk, topid)] for c in t.chunks
        ]:
            result_chunk_keys.append(n.key)
            dq_predecessors = deque([n])
            while dq_predecessors:
                current = dq_predecessors.popleft()
                reserve_chunk.update(n.op.outputs)
                predecessors = chunk_graph.predecessors(current)
                dq_predecessors.extend(
                    [p for p in predecessors if p not in reserve_chunk]
                )
                reserve_chunk.update(predecessors)
    # delete redundant chunk
    for n in list(chunk_graph.iter_nodes()):
        if n not in reserve_chunk:
            chunk_graph.remove_node(n)
        elif isinstance(n.op, Fetch):
            chunk_graph.remove_node(n)

    if compose:
        chunk_graph.compose(keys=result_chunk_keys)

    for tk, topid in tileable_key_opid_to_tiled:
        if tk not in self._target_tileable_chunk_ops:
            continue
        for n in tileable_key_opid_to_tiled[(tk, topid)][-1].chunks:
            self._terminal_chunk_op_tileable[n.op.key].add(tk)
            self._target_tileable_chunk_ops[tk].add(n.op.key)

    # sync chunk graph to kv store
    if self._kv_store_ref is not None:
        graph_path = "/sessions/%s/graphs/%s" % (self._session_id, self._graph_key)
        self._kv_store_ref.write(
            "%s/chunk_graph" % graph_path,
            serialize_graph(chunk_graph, compress=True),
            _tell=True,
            _wait=False,
        )

    self._chunk_graph_cache = chunk_graph
    for n in self._chunk_graph_cache:
        self._op_key_to_chunk[n.op.key].append(n)