# Buggy code
def estimate_fuse_size(ctx, op):
    from ...graph import DAG
    from ...executor import Executor

    chunk = op.outputs[0]
    dag = DAG()
    size_ctx = dict()
    keys = set(c.key for c in chunk.composed)
    for c in chunk.composed:
        dag.add_node(c)
        for inp in c.inputs:
            if inp.key not in keys:
                size_ctx[inp.key] = ctx[inp.key]
            if inp not in dag:
                dag.add_node(inp)
            dag.add_edge(inp, c)

    executor = Executor(storage=size_ctx)
    output_keys = [o.key for o in op.outputs]
    results = executor.execute_graph(dag, output_keys, mock=True, no_intermediate=True)
    ctx.update(zip(output_keys, results))

    # update with the maximal memory cost during the whole execution
    total_mem = sum(ctx[key][1] for key in output_keys)
    if total_mem:
        for key in output_keys:
            r = ctx[key]
            ctx[key] = (r[0], max(r[1], r[1] * executor.mock_max_memory // total_mem))


# Traceback (TypeError)
In [1]: from mars.session import new_session

In [2]: import mars.dataframe as md

In [3]: new_session(backend='ray').as_default()
2020-09-01 20:05:51,291	INFO resource_spec.py:231 -- Starting Ray with 5.08 GiB memory available for workers and up to 2.56 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).
2020-09-01 20:05:51,883	INFO services.py:1193 -- View the Ray dashboard at localhost:8265
Out[3]: <mars.session.Session at 0x7fc51364fb50>

In [4]: df = md.read_csv('Downloads/ratings.csv')

In [5]: df.groupby('userId').agg({'rating': ['min', 'max', 'mean', 'std']}).exec
...: ute()
---------------------------------------------------------------------------
RayTaskError(TypeError)                   Traceback (most recent call last)
<ipython-input-5-180cc92d1395> in <module>
----> 1 df.groupby('userId').agg({'rating': ['min', 'max', 'mean', 'std']}).execute()

~/Workspace/mars/mars/core.py in execute(self, session, **kw)
576
577     def execute(self, session=None, **kw):
--> 578         self._data.execute(session, **kw)
579         return self
580

~/Workspace/mars/mars/core.py in execute(self, session, **kw)
364
365         # no more fetch, thus just fire run
--> 366         session.run(self, **kw)
367         # return Tileable or ExecutableTuple itself
368         return self

~/Workspace/mars/mars/session.py in run(self, *tileables, **kw)
478         tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
479                           for t in tileables)
--> 480         result = self._sess.run(*tileables, **kw)
481
482         for t in tileables:

~/Workspace/mars/mars/ray/core.py in run(self, *tileables, **kw)
188         if 'n_parallel' not in kw:  # pragma: no cover
189             kw['n_parallel'] = ray.cluster_resources()['CPU']
--> 190         return self._executor.execute_tileables(tileables, **kw)
191
192     def __enter__(self):

~/Workspace/mars/mars/utils.py in _inner(*args, **kwargs)
437         def _inner(*args, **kwargs):
438             with self:
--> 439                 return func(*args, **kwargs)
440
441         return _inner

~/Workspace/mars/mars/executor.py in execute_tileables(self, tileables, fetch, n_parallel, n_thread, print_progress, mock, compose, name)
879                                    n_parallel=n_parallel or n_thread,
880                                    print_progress=print_progress, mock=mock,
--> 881                                    chunk_result=chunk_result)
882
883                 # update shape of tileable and its chunks whatever it's successful or not

~/Workspace/mars/mars/executor.py in execute_graph(self, graph, keys, n_parallel, print_progress, mock, no_intermediate, compose, retval, chunk_result)
691             print_progress=print_progress, mock=mock, mock_max_memory=self._mock_max_memory,
692             fetch_keys=fetch_keys, no_intermediate=no_intermediate)
--> 693         res = graph_execution.execute(retval)
694         self._mock_max_memory = max(self._mock_max_memory, graph_execution._mock_max_memory)
695         if mock:

~/Workspace/mars/mars/executor.py in execute(self, retval)
572         # wait until all the futures completed
573         for future in executed_futures:
--> 574             future.result()
575
576         if retval:

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
426                 raise CancelledError()
427             elif self._state == FINISHED:
--> 428                 return self.__get_result()
429
430             self._condition.wait(timeout)

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
382     def __get_result(self):
383         if self._exception:
--> 384             raise self._exception
385         else:
386             return self._result

~/miniconda3/lib/python3.7/concurrent/futures/thread.py in run(self)
55
56         try:
---> 57             result = self.fn(*self.args, **self.kwargs)
58         except BaseException as exc:
59             self.future.set_exception(exc)

~/Workspace/mars/mars/utils.py in _inner(*args, **kwargs)
437         def _inner(*args, **kwargs):
438             with self:
--> 439                 return func(*args, **kwargs)
440
441         return _inner

~/Workspace/mars/mars/executor.py in _execute_operand(self, op)
444             # so we pass the first operand's first output to Executor.handle
445             first_op = ops[0]
--> 446             self.handle_op(first_op, results, self._mock)
447
448             # update maximal memory usage during execution

~/Workspace/mars/mars/ray/core.py in handle_op(self, *args, **kw)
66 class GraphExecutionForRay(GraphExecution):
67     def handle_op(self, *args, **kw):
---> 68         return RayExecutor.handle(*args, **kw)
69
70

~/Workspace/mars/mars/ray/core.py in handle(cls, op, results, mock)
147
148         try:
--> 149             return ray.get(build_remote_funtion(runner).remote(results, op))
150         except NotImplementedError:
151             for op_cls in mapper.keys():

~/miniconda3/lib/python3.7/site-packages/ray/worker.py in get(object_refs, timeout)
1536                     worker.core_worker.dump_object_store_memory_usage()
1537                 if isinstance(value, RayTaskError):
-> 1538                     raise value.as_instanceof_cause()
1539                 else:
1540                     raise value

RayTaskError(TypeError): ray::mars.ray.core.remote_runner() (pid=31351, ip=30.225.12.80)
File "python/ray/_raylet.pyx", line 479, in ray._raylet.execute_task
File "/Users/qinxuye/Workspace/mars/mars/ray/core.py", line 144, in remote_runner
return func(results, op)
File "/Users/qinxuye/Workspace/mars/mars/dataframe/datasource/read_csv.py", line 322, in execute
df = cls._cudf_read_csv(op) if op.gpu else cls._pandas_read_csv(f, op)
File "/Users/qinxuye/Workspace/mars/mars/dataframe/datasource/read_csv.py", line 273, in _pandas_read_csv
dtype=dtypes.to_dict(), nrows=op.nrows, **csv_kwargs)
TypeError: parser_f() got an unexpected keyword argument 'outputs_ref'

# Fixed code
def estimate_fuse_size(ctx, op):
    from ...graph import DAG
    from ...executor import Executor
    from ...utils import build_fetch_chunk

    chunk = op.outputs[0]
    dag = DAG()
    size_ctx = dict()
    keys = set(c.key for c in chunk.composed)
    for c in chunk.composed:
        dag.add_node(c)
        for inp in c.inputs:
            if inp.key not in keys:
                size_ctx[inp.key] = ctx[inp.key]
                inp = build_fetch_chunk(inp).data
            if inp not in dag:
                dag.add_node(inp)
            dag.add_edge(inp, c)

    executor = Executor(storage=size_ctx)
    output_keys = [o.key for o in op.outputs]
    results = executor.execute_graph(dag, output_keys, mock=True, no_intermediate=True)
    ctx.update(zip(output_keys, results))

    # update with the maximal memory cost during the whole execution
    total_mem = sum(ctx[key][1] for key in output_keys)
    if total_mem:
        for key in output_keys:
            r = ctx[key]
            ctx[key] = (r[0], max(r[1], r[1] * executor.mock_max_memory // total_mem))