Repository: dask/dask

Issue:
DataFrame.to_bag raises PicklingError: Can't pickle Pandas
``` python
>>> df = dask.dataframe.read_csv('qq.csv')
>>> df.to_bag().take(5)

/miniconda3/envs/BigGBM/lib/python3.5/site-packages/dask/bag/core.py in take(self, k, compute)
    926         b = Bag(merge(self.dask, dsk), name, 1)
    927         if compute:
--> 928             return tuple(b.compute())
    929         else:
    930             return b

/miniconda3/envs/BigGBM/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
     84             Extra keywords to forward to the scheduler ``get`` function.
     85         """
---> 86         return compute(self, **kwargs)[0]
     87 
     88     @classmethod

/miniconda3/envs/BigGBM/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
    177         dsk = merge(var.dask for var in variables)
    178     keys = [var._keys() for var in variables]
--> 179     results = get(dsk, keys, **kwargs)
    180 
    181     results_iter = iter(results)

/miniconda3/envs/BigGBM/lib/python3.5/site-packages/dask/multiprocessing.py in get(dsk, keys, num_workers, func_loads, func_dumps, optimize_graph, **kwargs)
     81         # Run
     82         result = get_async(apply_async, len(pool._pool), dsk3, keys,
---> 83                            queue=queue, get_id=_process_get_id, **kwargs)
     84     finally:
     85         if cleanup:

/miniconda3/envs/BigGBM/lib/python3.5/site-packages/dask/async.py in get_async(apply_async, num_workers, dsk, result, cache, queue, get_id, raise_on_exception, rerun_exceptions_locally, callbacks, **kwargs)
    482                 _execute_task(task, data)  # Re-execute locally
    483             else:
--> 484                 raise(remote_exception(res, tb))
    485         state['cache'][key] = res
    486         finish_task(dsk, key, state, results, keyorder.get)

PicklingError: Can't pickle <class 'pandas.core.frame.Pandas'>: attribute lookup Pandas on pandas.core.frame failed

Traceback
---------
  File "/miniconda3/envs/BigGBM/lib/python3.5/site-packages/dask/async.py", line 277, in execute_task
    queue.put(result)
  File "<string>", line 2, in put
  File "/miniconda3/envs/BigGBM/lib/python3.5/multiprocessing/managers.py", line 716, in _callmethod
    conn.send((self._id, methodname, args, kwds))
  File "/miniconda3/envs/BigGBM/lib/python3.5/multiprocessing/connection.py", line 206, in send
    self._send_bytes(ForkingPickler.dumps(obj))
  File "/miniconda3/envs/BigGBM/lib/python3.5/multiprocessing/reduction.py", line 50, in dumps
    cls(buf, protocol).dump(obj)
```

Am I doing something wrong?

cc @jcrist 



Discussion:
Reproduced on my machine.  Seems like a genuine bug.  Something must not be serializable under `cloudpickle`. My first guess would some part of one of the lambdas in `to_bag`

``` python
In [19]: df.to_bag().dask
Out[19]: 
{'read-file-block-52bdf4b1da1488a4620659c71945e39e-0': (<function dask.bytes.local.read_block_from_file>,
  'foo.csv',
  0,
  64000000,
  b'\n',
  None),
 ('to_bag-41a74cfaeeb0a433fdb53eafaf2a3001',
  0): (<function dask.dataframe.io.to_bag.<locals>.<lambda>>, ('from-delayed-354f4d4f386c04edafb423ffff48f9c6',
   0)),
 ('from-delayed-354f4d4f386c04edafb423ffff48f9c6',
  0): 'bytes_read_csv-0ffdd53faad8ab3b93f648a20c50be5c',
 'bytes_read_csv-0ffdd53faad8ab3b93f648a20c50be5c': (<function dask.compatibility.apply>,
  <function dask.dataframe.csv.bytes_read_csv>,
  ['read-file-block-52bdf4b1da1488a4620659c71945e39e-0',
   b'a,b\n',
   (dict, []),
   (dict, [['b', dtype('int64')], ['a', dtype('int64')]]),
   (list, ['a', 'b'])],
  (dict, [['enforce', False], ['write_header', False]]))}
```

It's because itertuples now returns a namedtuple instead of a normal tuple. namedtuples aren't serializable under cloudpickle. Could cast to normal tuples, fix the issue in cloudpickle, make it a real class in pandas, or punt on this one. My vote is cast to tuples and fix in Pandas.


Resolving patch:
diff --git a/dask/dataframe/io.py b/dask/dataframe/io.py
--- a/dask/dataframe/io.py
+++ b/dask/dataframe/io.py
@@ -735,16 +735,20 @@ def to_csv(df, filename, name_function=None, compression=None, get=None, compute
         return delayed([Delayed(key, [dsk]) for key in keys])
 
 
+def _df_to_bag(df, index=False):
+    if isinstance(df, pd.DataFrame):
+        return list(map(tuple, df.itertuples(index)))
+    elif isinstance(df, pd.Series):
+        return list(df.iteritems()) if index else list(df)
+
+
 def to_bag(df, index=False):
     from ..bag.core import Bag
-    if isinstance(df, DataFrame):
-        func = lambda df: list(df.itertuples(index))
-    elif isinstance(df, Series):
-        func = (lambda df: list(df.iteritems())) if index else list
-    else:
+    if not isinstance(df, (DataFrame, Series)):
         raise TypeError("df must be either DataFrame or Series")
     name = 'to_bag-' + tokenize(df, index)
-    dsk = dict(((name, i), (func, block)) for (i, block) in enumerate(df._keys()))
+    dsk = dict(((name, i), (_df_to_bag, block, index))
+               for (i, block) in enumerate(df._keys()))
     dsk.update(df._optimize(df.dask, df._keys()))
     return Bag(dsk, name, df.npartitions)
 


Test patch:
diff --git a/dask/dataframe/tests/test_io.py b/dask/dataframe/tests/test_io.py
--- a/dask/dataframe/tests/test_io.py
+++ b/dask/dataframe/tests/test_io.py
@@ -1270,10 +1270,10 @@ def test_to_bag():
                      index=pd.Index([1., 2., 3., 4.], name='ind'))
     ddf = dd.from_pandas(a, 2)
 
-    assert ddf.to_bag().compute(get=get_sync) == list(a.itertuples(False))
-    assert ddf.to_bag(True).compute(get=get_sync) == list(a.itertuples(True))
-    assert ddf.x.to_bag(True).compute(get=get_sync) == list(a.x.iteritems())
-    assert ddf.x.to_bag().compute(get=get_sync) == list(a.x)
+    assert ddf.to_bag().compute() == list(a.itertuples(False))
+    assert ddf.to_bag(True).compute() == list(a.itertuples(True))
+    assert ddf.x.to_bag(True).compute() == list(a.x.iteritems())
+    assert ddf.x.to_bag().compute() == list(a.x)
 
 
 @pytest.mark.xfail(reason='we might want permissive behavior here')