Repository: dask/dask

Issue:
df.to_parquet throws TypeError: makedirs() got an unexpected keyword argument 'exist_ok'
HI,
I have a dask data-frame which I am trying to convert to parquet. 

```
import os, datetime
def createNewDir(basePath=PARQUET_FILES):
    parquet_dir = os.path.join(basePath, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
    os.makedirs(parquet_dir)
    return parquet_dir


```
`X_df.to_parquet(parquet_dir, compression='SNAPPY') `

I am using python 2.7 and this is the stack trace: 

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-26-a25af0083707> in <module>()
      4 # fastparquet.write(parquet_dir, X_df, file_scheme='hive',compression="SNAPPY")
      5 
----> 6 X_df.to_parquet('.', compression='SNAPPY')

/usr/local/lib/python2.7/dist-packages/dask/dataframe/core.pyc in to_parquet(self, path, *args, **kwargs)
    944         """ See dd.to_parquet docstring for more information """
    945         from .io import to_parquet
--> 946         return to_parquet(path, self, *args, **kwargs)
    947 
    948     def to_csv(self, filename, **kwargs):

/usr/local/lib/python2.7/dist-packages/dask/dataframe/io/parquet.pyc in to_parquet(path, df, compression, write_index, has_nulls, fixed_text, object_encoding, storage_options, append, ignore_divisions)
    223     myopen = OpenFileCreator(path, compression=None, text=False,
    224                              **(storage_options or {}))
--> 225     myopen.fs.mkdirs(path)
    226     sep = myopen.fs.sep
    227     metadata_fn = sep.join([path, '_metadata'])
/usr/local/lib/python2.7/dist-packages/dask/bytes/local.pyc in mkdirs(self, path)
     42         if path.startswith('file://'):
     43             path = path[len('file://'):]
---> 44         return os.makedirs(path, exist_ok=True)
     45 
     46     def open(self, path, mode='rb', **kwargs):

TypeError: makedirs() got an unexpected keyword argument 'exist_ok'



Discussion:
cc @martindurant 
I see this is not py2 compatible. Fix coming.
Question: do we not test parquet on py2? It should work from now.

Resolving patch:
diff --git a/dask/bytes/local.py b/dask/bytes/local.py
--- a/dask/bytes/local.py
+++ b/dask/bytes/local.py
@@ -41,7 +41,10 @@ def mkdirs(self, path):
         """Make any intermediate directories to make path writable"""
         if path.startswith('file://'):
             path = path[len('file://'):]
-        return os.makedirs(path, exist_ok=True)
+        try:
+            os.makedirs(path)
+        except OSError:
+            assert os.path.isdir(path)
 
     def open(self, path, mode='rb', **kwargs):
         """Make a file-like object
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -251,10 +251,10 @@ def to_parquet(path, df, compression=None, write_index=None, has_nulls=None,
             raise ValueError('Appended columns not the same.\n'
                              'New: {} | Previous: {}'
                              .format(pf.columns, list(df.columns)))
-        elif set(pf.dtypes.items()) != set(df.dtypes.items()):
+        elif set(pf.dtypes.items()) != set(df.dtypes.iteritems()):
             raise ValueError('Appended dtypes differ.\n{}'
                              .format(set(pf.dtypes.items()) ^
-                                     set(df.dtypes.items())))
+                                     set(df.dtypes.iteritems())))
         # elif fmd.schema != pf.fmd.schema:
         #    raise ValueError('Appended schema differs.')
         else:


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -1,3 +1,6 @@
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
 import os
 import numpy as np
 import pandas as pd
@@ -19,7 +22,7 @@ def test_local():
         data = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
                              'i64': np.arange(1000, dtype=np.int64),
                              'f': np.arange(1000, dtype=np.float64),
-                             'bhello': np.random.choice(['hello', 'you', 'people'], size=1000).astype("O")})
+                             'bhello': np.random.choice(['hello', 'yo', 'people'], size=1000).astype("O")})
         df = dd.from_pandas(data, chunksize=500)
 
         df.to_parquet(tmp, write_index=False, object_encoding='utf8')
@@ -144,7 +147,7 @@ def test_append():
         df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
                            'i64': np.arange(1000, dtype=np.int64),
                            'f': np.arange(1000, dtype=np.float64),
-                           'bhello': np.random.choice(['hello', 'you', 'people'],
+                           'bhello': np.random.choice(['hello', 'yo', 'people'],
                                                       size=1000).astype("O")})
         df.index.name = 'index'
 
@@ -164,7 +167,7 @@ def test_append_wo_index():
         df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
                            'i64': np.arange(1000, dtype=np.int64),
                            'f': np.arange(1000, dtype=np.float64),
-                           'bhello': np.random.choice(['hello', 'you', 'people'],
+                           'bhello': np.random.choice(['hello', 'yo', 'people'],
                                                       size=1000).astype("O")})
         half = len(df) // 2
         ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
@@ -190,7 +193,7 @@ def test_append_overlapping_divisions():
                            'i64': np.arange(1000, dtype=np.int64),
                            'f': np.arange(1000, dtype=np.float64),
                            'bhello': np.random.choice(
-                               ['hello', 'you', 'people'],
+                               ['hello', 'yo', 'people'],
                                size=1000).astype("O")})
         half = len(df) // 2
         ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
@@ -265,8 +268,8 @@ def test_read_parquet_custom_columns():
     (pd.DataFrame({'x': ['c', 'a', 'b']}), {'object_encoding': 'utf8'}, {}),
     (pd.DataFrame({'x': ['cc', 'a', 'bbb']}), {'object_encoding': 'utf8'}, {}),
     (pd.DataFrame({'x': [b'a', b'b', b'c']}), {'object_encoding': 'bytes'}, {}),
-    (pd.DataFrame({'x': pd.Categorical(['a', 'b', 'a'])}), {'object_encoding': 'utf8'},
-                                                           {'categories': ['x']}),
+    (pd.DataFrame({'x': pd.Categorical(['a', 'b', 'a'])}),
+     {'object_encoding': 'utf8'}, {'categories': ['x']}),
     (pd.DataFrame({'x': pd.Categorical([1, 2, 1])}), {}, {'categories': ['x']}),
     (pd.DataFrame({'x': list(map(pd.Timestamp, [3000, 2000, 1000]))}), {}, {}),
     (pd.DataFrame({'x': [3000, 2000, 1000]}).astype('M8[ns]'), {}, {}),