Repository: dask/dask

Issue:
.columns incorrect after merge
dask 0.7.6

```
In [42]: df1 = pd.DataFrame({'A' : [1,2,3], 'B' : list('abc'), 'C' : 'foo', 'D' : 1.0},columns=list('DCBA'))

In [43]: df1
Out[43]: 
   D    C  B  A
0  1  foo  a  1
1  1  foo  b  2
2  1  foo  c  3

In [44]: df2 = pd.DataFrame({'G' : [4,5], 'H' : 6.0, 'I' : 'bar', 'B' : list('ab')},columns=list('GHIB'))

In [45]: df2
Out[45]: 
   G  H    I  B
0  4  6  bar  a
1  5  6  bar  b

In [46]: pd.merge(df1,df2,on='B')
Out[46]: 
   D    C  B  A  G  H    I
0  1  foo  a  1  4  6  bar
1  1  foo  b  2  5  6  bar

In [47]: ddf = dd.from_pandas(df1,npartitions=3)

In [48]: dd.merge(ddf,df2,on='B').compute()
Out[48]: 
   D    C  B  A  G  H    I
0  1  foo  a  1  4  6  bar
1  1  foo  b  2  5  6  bar
```

```
# this is out of order w.r.t. the actual merged output
# and can affect future operations
In [49]: dd.merge(ddf,df2,on='B').columns  
Out[49]: ('D', 'C', 'A', 'G', 'H', 'I', 'B')
```

This might be because setting .columns doesn't have any effect

```
In [53]: df1 = pd.DataFrame({'A' : [1,2,3], 'B' : list('abc'), 'C' : 'foo', 'D' : 1.0},columns=list('DCBA'))

In [54]: ddf = dd.from_pandas(df1,npartitions=3)

In [55]: ddf
Out[55]: dd.DataFrame<from_pandas-668e7ccb027a1a83b01d197ba0395de9, divisions=(0, 1, 2, 2)>

In [56]: ddf.compute()
Out[56]: 
   D    C  B  A
0  1  foo  a  1
1  1  foo  b  2
2  1  foo  c  3

In [57]: ddf.columns = list('ABCD')

In [58]: ddf.compute()
Out[58]: 
   D    C  B  A
0  1  foo  a  1
1  1  foo  b  2
2  1  foo  c  3
```



Discussion:
A few things:
- Previously you couldn't set columns using `df.columns = ...`, as this would mutate the `dd.DataFrame`, but not the underlying computational graph. This was fixed in #961.
- The actual computation result is correct, but the order of the columns on our proxy aren't consistent with the order in the actual result. Unfortunately inferring this is proving difficult, as pandas has inconsistent behavior. We use an empty dataframe to hold the metadata, but executing the merge command on the empty frame results in a different column ordering than a non-empty frame:

``` python
In [10]: pd.merge(df1, df2, on='B').columns
Out[10]: Index([u'D', u'C', u'B', u'A', u'G', u'H', u'I'], dtype='object')

In [11]: pd.merge(df1.iloc[:0], df2.iloc[:0], on='B').columns
Out[11]: Index([u'D', u'C', u'A', u'G', u'H', u'I', u'B'], dtype='object')
```

It's not clear to me what the best path forward here is.

This comes up also with `.apply` where we would really like to know how a function behaves on a mostly empty dataframe.  I'm tempted to build a function that constructs a very small but non-empty dataframe with fake data.

Agreed I would actually do the calculation on a sample

Well, to be clear, we often don't actually have a sample of the data at this stage.  We would need to generate fake data.


Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -30,6 +30,7 @@
                      pseudorandom, derived_from, different_seeds, funcname)
 from ..base import Base, compute, tokenize, normalize_token
 from ..async import get_sync
+from .utils import nonempty_sample_df
 
 no_default = '__no_default__'
 return_scalar = '__return_scalar__'
@@ -1373,6 +1374,11 @@ def _constructor_sliced(self):
     def _constructor(self):
         return DataFrame
 
+    @property
+    def _pd_nonempty(self):
+        """ A Pandas dataframe with the same metadata but fake data."""
+        return nonempty_sample_df(self._pd)
+
     @property
     def columns(self):
         return self._pd.columns
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -203,8 +203,8 @@ def join_indexed_dataframes(lhs, rhs, how='left', lsuffix='', rsuffix=''):
     (lhs, rhs), divisions, parts = align_partitions(lhs, rhs)
     divisions, parts = require(divisions, parts, required[how])
 
-    left_empty = lhs._pd
-    right_empty = rhs._pd
+    left_empty = lhs._pd_nonempty
+    right_empty = rhs._pd_nonempty
 
     name = 'join-indexed-' + tokenize(lhs, rhs, how, lsuffix, rsuffix)
 
@@ -270,7 +270,7 @@ def hash_join(lhs, left_on, rhs, right_on, how='inner',
         right_index = False
 
     # dummy result
-    dummy = pd.merge(lhs._pd, rhs._pd, how, None,
+    dummy = pd.merge(lhs._pd_nonempty, rhs._pd_nonempty, how, None,
                      left_on=left_on, right_on=right_on,
                      left_index=left_index, right_index=right_index,
                      suffixes=suffixes)
@@ -299,7 +299,7 @@ def hash_join(lhs, left_on, rhs, right_on, how='inner',
 
 
 def single_partition_join(left, right, **kwargs):
-    meta = pd.merge(left._pd, right._pd, **kwargs)
+    meta = pd.merge(left._pd_nonempty, right._pd_nonempty, **kwargs)
     name = 'merge-' + tokenize(left, right, **kwargs)
     if left.npartitions == 1:
         left_key = first(left._keys())
@@ -623,4 +623,3 @@ def melt(frame, id_vars=None, value_vars=None, var_name=None,
                                 value_vars=value_vars,
                                 var_name=var_name, value_name=value_name,
                                 col_level=col_level, token='melt')
-
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
 from collections import Iterator
+import datetime
 import sys
 
 import numpy as np
@@ -9,6 +10,21 @@
 
 from dask.async import get_sync
 
+# Pandas <0.17 doesn't have the datetime64+tz extension dtype
+if pd.__version__ > '0.17.0':
+    is_datetime64tz_dtype = pd.core.common.is_datetime64tz_dtype
+else:
+    def is_datetime64tz_dtype(x):
+        return False
+
+# Pandas <0.18 doesn't have the `is_extension_type` function
+if pd.__version__ > '0.18.0':
+    is_extension_type = pd.core.common.is_extension_type
+else:
+    def is_extension_type(x):
+        return False
+
+
 
 def shard_df_on_index(df, divisions):
     """ Shard a DataFrame by ranges on its index
@@ -84,12 +100,59 @@ def unique(divisions):
         return np.unique(divisions)
     if isinstance(divisions, pd.Categorical):
         return pd.Categorical.from_codes(np.unique(divisions.codes),
-                divisions.categories, divisions.ordered)
+            divisions.categories, divisions.ordered)
     if isinstance(divisions, (tuple, list, Iterator)):
         return tuple(toolz.unique(divisions))
     raise NotImplementedError()
 
 
+_simple_fake_mapping = {
+    'b': True,   # boolean
+    'i': -1,   # signed int
+    'u': 1,   # unsigned int
+    'f': 1.0,   # float
+    'c': complex(1, 0),   # complex
+    'S': b'foo',   # bytestring
+    'U': u'foo',   # unicode string
+    'V': np.array([b' '], dtype=np.void)[0],  # void
+    'M': np.datetime64('1970-01-01'),
+    'm': np.timedelta64(1, 'D'),
+}
+
+
+def nonempty_sample_df(empty):
+    """ Create a dataframe from the given empty dataframe that contains one
+    row of fake data (generated from the empty dataframe's dtypes).
+    """
+    nonempty = {}
+    idx = pd.RangeIndex(start=0, stop=1, step=1)
+    for key, dtype in empty.dtypes.iteritems():
+        if is_datetime64tz_dtype(dtype):
+            entry = pd.Timestamp('1970-01-01', tz='America/New_York')
+        elif pd.core.common.is_categorical_dtype(dtype):
+            accessor = empty[key].cat
+            example = accessor.categories[0]
+            cat = pd.Categorical([example], categories=accessor.categories,
+                                 ordered=accessor.ordered)
+            entry = pd.Series(cat, name=key)
+        elif dtype.kind in _simple_fake_mapping:
+            entry = _simple_fake_mapping[dtype.kind]
+        elif is_extension_type(dtype):
+            raise TypeError("Can't handle extension dtype: {}".format(dtype))
+        elif dtype.name == 'object':
+            entry = 'foo'
+        else:
+            raise TypeError("Can't handle dtype: {}".format(dtype))
+
+        if not isinstance(entry, pd.Series):
+            entry = pd.Series([entry], name=key, index=idx)
+
+        nonempty[key] = entry
+
+    df = pd.DataFrame(nonempty)
+    return df
+
+
 ###############################################################
 # Testing
 ###############################################################


Test patch:
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -556,3 +556,19 @@ def test_cheap_single_partition_merge():
 
     list_eq(aa.merge(bb, on='x', how='inner'),
             a.merge(b, on='x', how='inner'))
+
+
+@pytest.mark.parametrize('lhs', [pd.DataFrame({'A': [1, 2, 3],
+                                               'B': list('abc'),
+                                               'C': 'foo',
+                                               'D': 1.0},
+                                              columns=list('DCBA'))])
+@pytest.mark.parametrize('rhs', [pd.DataFrame({'G': [4, 5],
+                                               'H': 6.0,
+                                               'I': 'bar',
+                                               'B': list('ab')},
+                                              columns=list('GHIB'))])
+def test_merge_maintains_columns(lhs, rhs):
+    ddf = dd.from_pandas(lhs, npartitions=1)
+    merged = dd.merge(ddf, rhs, on='B').compute()
+    assert tuple(merged.columns) == ('D', 'C', 'B', 'A', 'G', 'H', 'I')
diff --git a/dask/dataframe/tests/test_utils_dataframe.py b/dask/dataframe/tests/test_utils_dataframe.py
--- a/dask/dataframe/tests/test_utils_dataframe.py
+++ b/dask/dataframe/tests/test_utils_dataframe.py
@@ -1,5 +1,6 @@
+import numpy as np
 import pandas as pd
-from dask.dataframe.utils import shard_df_on_index
+from dask.dataframe.utils import shard_df_on_index, nonempty_sample_df
 
 
 def test_shard_df_on_index():
@@ -11,3 +12,26 @@ def test_shard_df_on_index():
     assert list(result[1].index) == [20, 30, 40]
     assert list(result[2].index) == [50, 60]
 
+
+def test_nonempty_sample_df():
+    df1 = pd.DataFrame({'A': pd.Categorical(['Alice', 'Bob', 'Carol']),
+                        'B': list('abc'),
+                        'C': 'bar',
+                        'D': 3.0,
+                        'E': pd.Timestamp('2016-01-01'),
+                        'F': pd.date_range('2016-01-01', periods=3,
+                                           tz='America/New_York'),
+                        'G': pd.Timedelta('1 hours'),
+                        'H': np.void(b' ')},
+                       columns=list('DCBAHGFE'))
+    df2 = df1.iloc[0:0]
+    df3 = nonempty_sample_df(df2)
+    assert df3['A'][0] == 'Alice'
+    assert df3['B'][0] == 'foo'
+    assert df3['C'][0] == 'foo'
+    assert df3['D'][0] == 1.0
+    assert df3['E'][0] == pd.Timestamp('1970-01-01 00:00:00')
+    assert df3['F'][0] == pd.Timestamp('1970-01-01 00:00:00',
+                                       tz='America/New_York')
+    assert df3['G'][0] == pd.Timedelta('1 days')
+    assert df3['H'][0] == 'foo'