Repository: dask/dask

Issue:
Bag.to_dataframe fails on scalar data
``` python
>>> dask.bag.from_sequence([1,2]).to_dataframe()

TypeError                                 Traceback (most recent call last)
<ipython-input-140-756073df0a62> in <module>()
----> 1 dask.bag.from_sequence([1,2]).to_dataframe()

/opt/conda/lib/python3.5/site-packages/dask/bag/core.py in to_dataframe(self, columns)
   1044 
   1045         return dd.DataFrame(merge(optimize(self.dask, self._keys()), dsk),
-> 1046                             name, columns, divisions)
   1047 
   1048     def to_imperative(self):

/opt/conda/lib/python3.5/site-packages/dask/dataframe/core.py in __new__(cls, dsk, name, columns, divisions)
   1495         result._name = name
   1496 
-> 1497         result._pd, result._known_dtype = cls._build_pd(columns)
   1498         result.divisions = tuple(divisions)
   1499         return result

/opt/conda/lib/python3.5/site-packages/dask/dataframe/core.py in _build_pd(cls, metadata)
    200         else:
    201             if np.isscalar(metadata) or metadata is None:
--> 202                 _pd = cls._partition_type([], name=metadata)
    203                 known_dtype = False
    204             elif isinstance(metadata, datetime):

TypeError: __init__() got an unexpected keyword argument 'name'
```



Discussion:
Currently `to_dataframe` is only intended to work when the element is a dict or tuple (as explained in [the docstring](http://dask.pydata.org/en/latest/bag-api.html#dask.bag.Bag.to_dataframe)). What is the expected behavior here, return a `dd.Series`? Without knowing the type of the elements, we can't pick a series or a dataframe easily at graph build time.

One option would be to return a `dask.Series` if `columns` is just a string (for the name of the series), otherwise return a dataframe (which would expect records as elements).

@jcrist The expected behavior: fail with a meaningful error.

Also, it seems that the `if` branch in `_build_pd` will fail on any data due to the fact that `cls._partiton_type` doesn't expect `name` keyword argument, but it is passed there explicitly.

A meaningful error is a good fix. The `_build_pd` stuff is being reworked in #1422, and is already fixed there.


Resolving patch:
diff --git a/dask/bag/core.py b/dask/bag/core.py
--- a/dask/bag/core.py
+++ b/dask/bag/core.py
@@ -1004,16 +1004,20 @@ def groupby(self, grouper, method=None, npartitions=None, blocksize=2**20,
     def to_dataframe(self, columns=None):
         """ Convert Bag to dask.dataframe
 
-        Bag should contain tuple or dict records.
-
-        Provide ``columns=`` keyword arg to specify column names.
+        Bag should contain tuples, dict records, or scalars.
 
         Index will not be particularly meaningful.  Use ``reindex`` afterwards
         if necessary.
 
+        Parameters
+        ----------
+        columns : list or pandas.DataFrame, optional
+            If a list, provides the desired column names. If a 
+            ``pandas.DataFrame``, it should mirror the column names and dtypes
+            of the output dataframe. If not provided, these will be computed.
+
         Examples
         --------
-
         >>> import dask.bag as db
         >>> b = db.from_sequence([{'name': 'Alice',   'balance': 100},
         ...                       {'name': 'Bob',     'balance': 200},
@@ -1029,21 +1033,20 @@ def to_dataframe(self, columns=None):
         """
         import pandas as pd
         import dask.dataframe as dd
-        if columns is None:
+        if isinstance(columns, pd.DataFrame):
+            meta = columns
+        else:
             head = self.take(1)[0]
-            if isinstance(head, dict):
-                columns = sorted(head)
-            elif isinstance(head, (tuple, list)):
-                columns = list(range(len(head)))
+            meta = pd.DataFrame([head], columns=columns)
+        columns = list(meta.columns)
         name = 'to_dataframe-' + tokenize(self, columns)
         DataFrame = partial(pd.DataFrame, columns=columns)
         dsk = dict(((name, i), (DataFrame, (list2, (self.name, i))))
                    for i in range(self.npartitions))
 
         divisions = [None] * (self.npartitions + 1)
-
         return dd.DataFrame(merge(optimize(self.dask, self._keys()), dsk),
-                            name, columns, divisions)
+                            name, meta, divisions)
 
     def to_imperative(self):
         warn("Deprecation warning: moved to to_delayed")


Test patch:
diff --git a/dask/bag/tests/test_bag.py b/dask/bag/tests/test_bag.py
--- a/dask/bag/tests/test_bag.py
+++ b/dask/bag/tests/test_bag.py
@@ -683,6 +683,16 @@ def test_to_dataframe():
     assert df2._name == b.to_dataframe()._name
     assert df2._name != df._name
 
+    meta = pd.DataFrame({'a': [1], 'b': [2]}).iloc[0:0]
+    df3 = b.to_dataframe(columns=meta)
+    assert df2._name == df3._name
+    assert (df3.compute().values == df2.compute().values).all()
+
+    b = db.from_sequence([1, 2, 3, 4, 5], npartitions=2)
+    df4 = b.to_dataframe()
+    assert len(df4.columns) == 1
+    assert list(df4.compute()) == list(pd.DataFrame(list(b)))
+
 
 def test_to_textfiles():
     b = db.from_sequence(['abc', '123', 'xyz'], npartitions=2)