Repository: dask/dask

Issue:
dask.dataframe.to_parquet fails when append=True
The following code generated an __ValueError: Appended columns not the same__ at the last line:

```
import numpy as np
import pandas as pd
from dask import dataframe as dd
df0 = pd.DataFrame({'lat':np.arange(0,10), 'lon':np.arange(10,20), 'value': np.arange(100,110)})
df1 = pd.DataFrame({'lat':np.arange(10,20), 'lon':np.arange(10,20), 'value': np.arange(120,130)})
dd_df0 = dd.from_pandas(df0, npartitions=1)
dd_df1 = dd.from_pandas(df1, npartitions=1)
dd.to_parquet(path = '~/test', df=dd_df0, partition_on=['lon'])
dd.to_parquet(path = '~/test', df=dd_df1, partition_on=['lon'], append=True)
```

The full error message is below:

```---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-67-30a65c031310> in <module>()
----> 1 dd.to_parquet(path = '~/test', df=dd_df1,  append=True)

/no_backup_ssd/p3env/lib/python3.5/site-packages/dask/dataframe/io/parquet.py in to_parquet(path, df, compression, write_index, has_nulls, fixed_text, object_encoding, storage_options, append, ignore_divisions, partition_on, compute)
    397             raise ValueError('Appended columns not the same.\n'
    398                              'New: {} | Previous: {}'
--> 399                              .format(pf.columns, list(df.columns)))
    400         elif set(pf.dtypes.items()) != set(df.dtypes.iteritems()):
    401             raise ValueError('Appended dtypes differ.\n{}'

ValueError: Appended columns not the same.
New: ['index', 'lat', 'value'] | Previous: ['index', 'lat', 'lon', 'value']
```

version number:
Python 3.5.2
pandas = 0.20.3
dask = 0.15.2
fastparquest = 0.1.2




Discussion:
Looks like a genuine bug: `lon` was removed from the columns list because it was used for partitioning. I'll look into it.
Thanks. Is my usage of the dask correct?
I would comment that the partitioning you chose will result in one row per file on disc, which is obviously not efficient or desirable. I expect this is because the example is contrived for the sake of simplicity. That aside, the code ought to work.

Resolving patch:
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -393,32 +393,35 @@ def to_parquet(path, df, compression=None, write_index=None, has_nulls=True,
         if pf.file_scheme != 'hive':
             raise ValueError('Requested file scheme is hive, '
                              'but existing file scheme is not.')
-        elif set(pf.columns) != set(df.columns):
+        elif ((set(pf.columns) != set(df.columns) - set(partition_on)) or
+              (set(partition_on) != set(pf.cats))):
             raise ValueError('Appended columns not the same.\n'
                              'New: {} | Previous: {}'
                              .format(pf.columns, list(df.columns)))
-        elif set(pf.dtypes.items()) != set(df.dtypes.iteritems()):
+        elif set(pf.dtypes[c] for c in pf.columns) != set(df[pf.columns].dtypes):
             raise ValueError('Appended dtypes differ.\n{}'
                              .format(set(pf.dtypes.items()) ^
                                      set(df.dtypes.iteritems())))
         # elif fmd.schema != pf.fmd.schema:
         #    raise ValueError('Appended schema differs.')
         else:
-            df = df[pf.columns]
+            df = df[pf.columns + partition_on]
 
         fmd = pf.fmd
         i_offset = fastparquet.writer.find_max_part(fmd.row_groups)
 
         if not ignore_divisions:
             minmax = fastparquet.api.sorted_partitioned_columns(pf)
-            divisions = list(minmax[index_col]['min']) + [
-                minmax[index_col]['max'][-1]]
-
-            if new_divisions[0] < divisions[-1]:
-                raise ValueError(
-                    'Appended divisions overlapping with the previous ones.\n'
-                    'New: {} | Previous: {}'
-                    .format(divisions[-1], new_divisions[0]))
+            if index_col in minmax:
+
+                divisions = list(minmax[index_col]['min']) + [
+                    minmax[index_col]['max'][-1]]
+
+                if new_divisions[0] < divisions[-1]:
+                    raise ValueError(
+                        'Appended divisions overlapping with the previous ones.'
+                        '\nNew: {} | Previous: {}'
+                        .format(divisions[-1], new_divisions[0]))
     else:
         i_offset = 0
 


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -208,6 +208,27 @@ def test_append():
         assert_eq(df, ddf3)
 
 
+def test_append_with_partition():
+    with tmpdir() as tmp:
+        df0 = pd.DataFrame({'lat': np.arange(0, 10), 'lon': np.arange(10, 20),
+                            'value': np.arange(100, 110)})
+        df0.index.name = 'index'
+        df1 = pd.DataFrame({'lat': np.arange(10, 20), 'lon': np.arange(10, 20),
+                            'value': np.arange(120, 130)})
+        df1.index.name = 'index'
+        dd_df0 = dd.from_pandas(df0, npartitions=1)
+        dd_df1 = dd.from_pandas(df1, npartitions=1)
+        dd.to_parquet(tmp, dd_df0, partition_on=['lon'])
+        dd.to_parquet(tmp, dd_df1, partition_on=['lon'], append=True,
+                      ignore_divisions=True)
+
+        out = dd.read_parquet(tmp).compute()
+    out['lon'] = out.lon.astype('int64')  # just to pass assert
+    # sort required since partitioning breaks index order
+    assert_eq(out.sort_values('value'), pd.concat([df0, df1])[out.columns],
+              check_index=False)
+
+
 def test_append_wo_index():
     """Test append with write_index=False."""
     with tmpdir() as tmp: