# Buggy code
def _dump_cache(self, session_id, graph_key, inproc_uid, save_sizes):
    """
    Dump calc results into shared cache or spill
    :param session_id: session id
    :param graph_key: key of the execution graph
    :param inproc_uid: uid of the InProcessCacheActor
    :param save_sizes: sizes of data
    """
    graph_record = self._graph_records[session_id, graph_key]
    chunk_keys = graph_record.chunk_targets
    calc_keys = list(save_sizes.keys())
    send_addresses = graph_record.send_addresses

    logger.debug(
        "Graph %s: Start putting %r into shared cache. Target actor uid %s.",
        graph_key,
        chunk_keys,
        inproc_uid,
    )
    self._update_state(session_id, graph_key, ExecutionState.STORING)

    raw_inproc_ref = self.ctx.actor_ref(inproc_uid)
    inproc_ref = self.promise_ref(raw_inproc_ref)

    if graph_record.stop_requested:
        logger.debug("Graph %s already marked for stop, quit.", graph_key)
        if (
            self._daemon_ref is None
            or self._daemon_ref.is_actor_process_alive(raw_inproc_ref)
        ) and self.ctx.has_actor(raw_inproc_ref):
            logger.debug("Try remove keys for graph %s.", graph_key)
            raw_inproc_ref.remove_cache(session_id, list(calc_keys), _tell=True)
        logger.debug("Graph %s already marked for stop, quit.", graph_key)
        raise ExecutionInterrupted

    self._chunk_holder_ref.unpin_chunks(
        graph_key, list(set(c.key for c in graph_record.graph)), _tell=True
    )
    self._dump_execution_states()

    if self._daemon_ref is not None and not self._daemon_ref.is_actor_process_alive(
        raw_inproc_ref
    ):
        raise WorkerProcessStopped

    def _cache_result(*_):
        self._result_cache[(session_id, graph_key)] = GraphResultRecord(save_sizes)

    if not send_addresses:
        # no endpoints to send, dump keys into shared memory and return
        logger.debug(
            "Worker graph %s(%s) finished execution. Dumping results into plasma...",
            graph_key,
            graph_record.op_string,
        )
        return inproc_ref.dump_cache(session_id, calc_keys, _promise=True).then(
            _cache_result
        )
    else:
        # dump keys into shared memory and send
        all_addresses = [
            {v} if isinstance(v, six.string_types) else set(v)
            for v in send_addresses.values()
        ]
        all_addresses = list(reduce(lambda a, b: a | b, all_addresses, set()))
        logger.debug(
            "Worker graph %s(%s) finished execution. Dumping results into plasma "
            "while actively transferring into %r...",
            graph_key,
            graph_record.op_string,
            all_addresses,
        )

        data_to_addresses = dict(
            (k, v) for k, v in send_addresses.items() if k in save_sizes
        )

        return (
            inproc_ref.dump_cache(session_id, calc_keys, _promise=True)
            .then(save_sizes.update)
            .then(
                lambda *_: functools.partial(
                    self._do_active_transfer, session_id, graph_key, data_to_addresses
                )
            )
            .then(_cache_result)
        )


# Traceback (ValueError)
500 GET /worker?endpoint
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/tornado/web.py", line 1592, in _execute
result = yield result
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1133, in run
value = future.result()
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1147, in run
yielded = self.gen.send(value)
File "/opt/conda/lib/python3.6/site-packages/bokeh/server/views/doc_handler.py", line 27, in get
template_variables=session.document.template_variables)
File "/opt/conda/lib/python3.6/site-packages/bokeh/embed/server.py", line 230, in server_html_page_for_session
template=template, template_variables=template_variables)
File "/opt/conda/lib/python3.6/site-packages/bokeh/embed/elements.py", line 133, in html_page_for_render_items
html = template.render(context)
File "/opt/conda/lib/python3.6/site-packages/jinja2/asyncsupport.py", line 76, in render
return original_render(self, *args, **kwargs)
File "/opt/conda/lib/python3.6/site-packages/jinja2/environment.py", line 1008, in render
return self.environment.handle_exception(exc_info, True)
File "/opt/conda/lib/python3.6/site-packages/jinja2/environment.py", line 780, in handle_exception
reraise(exc_type, exc_value, tb)
File "/opt/conda/lib/python3.6/site-packages/jinja2/_compat.py", line 37, in reraise
raise value.with_traceback(tb)
File "/home/admin/work/public-mars-0.2.0b1-cupid.zip/mars/web/templates/worker_detail.html", line 1, in top-level template code
{% extends "base.html" %}
File "/home/admin/work/public-mars-0.2.0b1-cupid.zip/mars/web/templates/base.html", line 65, in top-level template code
{% block body %}{% endblock %}
File "/home/admin/work/public-mars-0.2.0b1-cupid.zip/mars/web/templates/worker_detail.html", line 252, in block "body"
<td>{{ stats['min_est_finish_time'] | format_ts }}</td>
File "/home/admin/work/public-mars-0.2.0b1-cupid.zip/mars/web/server.py", line 49, in format_ts
return datetime.fromtimestamp(value).strftime('%Y-%m-%d %H:%M:%S')
ValueError: Invalid value NaN (not a number)
500 GET /worker?endpoint=10.101.216.148:53630 (10.101.202.142) 137.45ms

# Fixed code
def _dump_cache(self, session_id, graph_key, inproc_uid, save_sizes):
    """
    Dump calc results into shared cache or spill
    :param session_id: session id
    :param graph_key: key of the execution graph
    :param inproc_uid: uid of the InProcessCacheActor
    :param save_sizes: sizes of data
    """
    graph_record = self._graph_records[session_id, graph_key]
    chunk_keys = graph_record.chunk_targets
    calc_keys = list(save_sizes.keys())
    send_addresses = graph_record.send_addresses

    logger.debug(
        "Graph %s: Start putting %r into shared cache. Target actor uid %s.",
        graph_key,
        chunk_keys,
        inproc_uid,
    )
    self._update_state(session_id, graph_key, ExecutionState.STORING)

    raw_inproc_ref = self.ctx.actor_ref(inproc_uid)
    inproc_ref = self.promise_ref(raw_inproc_ref)

    if graph_record.stop_requested:
        logger.debug("Graph %s already marked for stop, quit.", graph_key)
        if (
            self._daemon_ref is None
            or self._daemon_ref.is_actor_process_alive(raw_inproc_ref)
        ) and self.ctx.has_actor(raw_inproc_ref):
            logger.debug("Try remove keys for graph %s.", graph_key)
            raw_inproc_ref.remove_cache(session_id, list(calc_keys), _tell=True)
        logger.debug("Graph %s already marked for stop, quit.", graph_key)
        raise ExecutionInterrupted

    self._chunk_holder_ref.unpin_chunks(
        graph_key, list(set(c.key for c in graph_record.graph)), _tell=True
    )
    self._dump_execution_states()

    if self._daemon_ref is not None and not self._daemon_ref.is_actor_process_alive(
        raw_inproc_ref
    ):
        raise WorkerProcessStopped

    def _cache_result(result_sizes):
        save_sizes.update(result_sizes)
        self._result_cache[(session_id, graph_key)] = GraphResultRecord(save_sizes)

    if not send_addresses:
        # no endpoints to send, dump keys into shared memory and return
        logger.debug(
            "Worker graph %s(%s) finished execution. Dumping results into plasma...",
            graph_key,
            graph_record.op_string,
        )
        return (
            inproc_ref.dump_cache(session_id, calc_keys, _promise=True)
            .then(_cache_result)
            .then(lambda *_: self._notify_successors(session_id, graph_key))
        )
    else:
        # dump keys into shared memory and send
        all_addresses = [
            {v} if isinstance(v, six.string_types) else set(v)
            for v in send_addresses.values()
        ]
        all_addresses = list(reduce(lambda a, b: a | b, all_addresses, set()))
        logger.debug(
            "Worker graph %s(%s) finished execution. Dumping results into plasma "
            "while actively transferring into %r...",
            graph_key,
            graph_record.op_string,
            all_addresses,
        )

        data_to_addresses = dict(
            (k, v) for k, v in send_addresses.items() if k in save_sizes
        )

        return (
            inproc_ref.dump_cache(session_id, calc_keys, _promise=True)
            .then(_cache_result)
            .then(lambda *_: self._notify_successors(session_id, graph_key))
            .then(
                lambda *_: functools.partial(
                    self._do_active_transfer, session_id, graph_key, data_to_addresses
                )
            )
        )