Repository: dask/dask

Issue:
BUG: pandas Parquet metadata is only being written to the _metadata file 
Encountered as part of https://github.com/dask/dask/issues/2439

Setup:

```
import pandas as pd
import dask.dataframe as dd
from dask.dataframe.io.parquet import read_parquet, to_parquet

df = pd.DataFrame({'x': [6, 2, 3, 4, 5],
                   'y': [1.0, 2.0, 1.0, 2.0, 1.0]},
                  index=pd.Index([10, 20, 30, 40, 50], name='myindex'))
ddf = dd.from_pandas(df, npartitions=3)
to_parquet('example', ddf)
```

And then:

```
In [10]: import pyarrow.parquet as pq

In [11]: pq.ParquetFile('example/part.0.parquet').metadata.metadata
Out[11]: {}

In [12]: pq.ParquetFile('example/_metadata').metadata.metadata
Out[12]: {b'pandas': b'{"index_columns": ["myindex"], "columns": [{"name": "myindex", "pandas_type": "int64", "numpy_dtype": "int64", "metadata": null}, {"name": "x", "pandas_type": "int64", "numpy_dtype": "int64", "metadata": null}, {"name": "y", "pandas_type": "float64", "numpy_dtype": "float64", "metadata": null}], "pandas_version": "0.20.2"}'}
```

It seems OK to override the partition metadata with the `_metadata` file (we can handle this on the Arrow side for directory reads), but it may be a good idea to replicate the pandas metadata to all of the individual data set pieces


Discussion:
See https://issues.apache.org/jira/browse/ARROW-1103
Good idea, metadata should be in every data file (as the parquet schema already is)
@martindurant there is also some implementation slippage, the parameter for the NumPy type is `numpy_type` per:

https://github.com/pandas-dev/pandas/blob/master/doc/source/developer.rst
We should probably set up an integration test repo someplace that is independent from dask, to make these issues easier to test. What do you think?
You mean a fastparquet/arrow/other-parquet cross-library integration tests? That sounds like a decent idea, but of course someone would have to put effort into it. One might argue that dask's needs are more focussed.
I should think that Dask users would be concerned with correctness at the single Parquet file level. The intent would be to consolidate our unit testing efforts to put all "problematic pandas data" in one place so we can verify that data written by one implementation can be accurately read by another. I am willing to spend some time with setting up a repo for this under pandas-dev
Sorry, accidental auto-close, dealing with numpy_type only.

Resolving patch:
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -1,10 +1,12 @@
 from __future__ import absolute_import, division, print_function
 
+import copy
 import json
 import warnings
 
 import numpy as np
 import pandas as pd
+from toolz import unique
 
 from ..core import DataFrame, Series
 from ..utils import UNKNOWN_CATEGORIES
@@ -13,34 +15,26 @@
 from ...delayed import delayed
 from ...bytes.core import get_fs_paths_myopen
 
-try:
-    import fastparquet
-    from fastparquet.core import read_row_group_file
-    from fastparquet.api import _pre_allocate
-    from fastparquet.util import check_column_names
-except ImportError:
-    fastparquet = False
-else:
-    @normalize_token.register(fastparquet.ParquetFile)
-    def normalize_ParquetFile(pf):
-        return (type(pf), pf.fn, pf.sep) + normalize_token(pf.open)
-
-try:
-    import pyarrow.parquet as pyarrow_parquet
-except ImportError:
-    pyarrow_parquet = False
-else:
-    @normalize_token.register(pyarrow_parquet.ParquetDataset)
-    def normalize_PyArrowParquetDataset(ds):
-        return (type(ds), ds.paths)
-
-
-def _meta_from_dtypes(to_read_columns, file_columns, file_dtypes):
+
+__all__ = ('read_parquet', 'to_parquet')
+
+
+def _meta_from_dtypes(to_read_columns, file_columns, file_dtypes, index_cols):
     meta = pd.DataFrame({c: pd.Series([], dtype=d)
                         for (c, d) in file_dtypes.items()},
                         columns=[c for c in file_columns
                                  if c in file_dtypes])
-    return meta[list(to_read_columns)]
+    df = meta[list(to_read_columns)]
+
+    if not index_cols:
+        return df
+    if not isinstance(index_cols, list):
+        index_cols = [index_cols]
+    df = df.set_index(index_cols)
+    if len(index_cols) == 1 and index_cols[0] == '__index_level_0__':
+        df.index.name = None
+    return df
+
 
 # ----------------------------------------------------------------------
 # Fastparquet interface
@@ -48,6 +42,8 @@ def _meta_from_dtypes(to_read_columns, file_columns, file_dtypes):
 
 def _read_fastparquet(fs, paths, myopen, columns=None, filters=None,
                       categories=None, index=None, storage_options=None):
+    import fastparquet
+    from fastparquet.util import check_column_names
     if filters is None:
         filters = []
 
@@ -95,15 +91,13 @@ def _read_fastparquet(fs, paths, myopen, columns=None, filters=None,
     dtypes = pf._dtypes(categories)
 
     meta = _meta_from_dtypes(all_columns, tuple(pf.columns + list(pf.cats)),
-                             dtypes)
+                             dtypes, index_col)
 
     for cat in categories:
         if cat in meta:
             meta[cat] = pd.Series(pd.Categorical([],
-                                  categories=[UNKNOWN_CATEGORIES]))
-
-    if index_col:
-        meta = meta.set_index(index_col)
+                                  categories=[UNKNOWN_CATEGORIES]),
+                                  index=meta.index)
 
     if out_type == Series:
         assert len(meta.columns) == 1
@@ -141,6 +135,8 @@ def _read_fastparquet(fs, paths, myopen, columns=None, filters=None,
 
 def _read_parquet_row_group(open, fn, index, columns, rg, series, categories,
                             schema, cs, dt, scheme, *args):
+    from fastparquet.api import _pre_allocate
+    from fastparquet.core import read_row_group_file
     if not isinstance(columns, (tuple, list)):
         columns = (columns,)
         series = True
@@ -156,12 +152,131 @@ def _read_parquet_row_group(open, fn, index, columns, rg, series, categories,
         return df
 
 
+def _write_partition_fastparquet(df, fs, path, filename, fmd, compression,
+                                 partition_on):
+    from fastparquet.writer import partition_on_columns, make_part_file
+    # Fastparquet mutates this in a non-threadsafe manner. For now we just copy
+    # it before forwarding to fastparquet.
+    fmd = copy.copy(fmd)
+    if not len(df):
+        # Write nothing for empty partitions
+        rgs = None
+    elif partition_on:
+        rgs = partition_on_columns(df, partition_on, path, filename, fmd,
+                                   fs.sep, compression, fs.open, fs.mkdirs)
+    else:
+        # Fastparquet current doesn't properly set `num_rows` in the output
+        # metadata. Set it here to fix that.
+        fmd.num_rows = len(df)
+        with fs.open(fs.sep.join([path, filename]), 'wb') as fil:
+            rgs = make_part_file(fil, df, fmd.schema, compression=compression,
+                                 fmd=fmd)
+    return rgs
+
+
+def _write_fastparquet(df, path, write_index=None, append=False,
+                       ignore_divisions=False, partition_on=None,
+                       storage_options=None, compression=None, **kwargs):
+    import fastparquet
+
+    fs, paths, open_with = get_fs_paths_myopen(path, None, 'wb',
+                                               **(storage_options or {}))
+    fs.mkdirs(path)
+    sep = fs.sep
+
+    object_encoding = kwargs.pop('object_encoding', 'utf8')
+    if object_encoding == 'infer' or (isinstance(object_encoding, dict) and
+                                      'infer' in object_encoding.values()):
+        raise ValueError('"infer" not allowed as object encoding, '
+                         'because this required data in memory.')
+
+    divisions = df.divisions
+    if write_index is True or write_index is None and df.known_divisions:
+        df = df.reset_index()
+        index_cols = [df.columns[0]]
+    else:
+        ignore_divisions = True
+        index_cols = []
+
+    if append:
+        pf = fastparquet.api.ParquetFile(path, open_with=open_with, sep=sep)
+        if pf.file_scheme not in ['hive', 'empty', 'flat']:
+            raise ValueError('Requested file scheme is hive, '
+                             'but existing file scheme is not.')
+        elif ((set(pf.columns) != set(df.columns) - set(partition_on)) or
+              (set(partition_on) != set(pf.cats))):
+            raise ValueError('Appended columns not the same.\n'
+                             'New: {} | Previous: {}'
+                             .format(pf.columns, list(df.columns)))
+        elif set(pf.dtypes[c] for c in pf.columns) != set(df[pf.columns].dtypes):
+            raise ValueError('Appended dtypes differ.\n{}'
+                             .format(set(pf.dtypes.items()) ^
+                                     set(df.dtypes.iteritems())))
+        else:
+            df = df[pf.columns + partition_on]
+
+        fmd = pf.fmd
+        i_offset = fastparquet.writer.find_max_part(fmd.row_groups)
+
+        if not ignore_divisions:
+            minmax = fastparquet.api.sorted_partitioned_columns(pf)
+            old_end = minmax[index_cols[0]]['max'][-1]
+            if divisions[0] < old_end:
+                raise ValueError(
+                    'Appended divisions overlapping with the previous ones.\n'
+                    'New: {} | Previous: {}'.format(old_end, divisions[0]))
+    else:
+        fmd = fastparquet.writer.make_metadata(df._meta,
+                                               object_encoding=object_encoding,
+                                               index_cols=index_cols,
+                                               ignore_columns=partition_on,
+                                               **kwargs)
+        i_offset = 0
+
+    filenames = ['part.%i.parquet' % (i + i_offset)
+                 for i in range(df.npartitions)]
+
+    write = delayed(_write_partition_fastparquet)
+    writes = [write(part, fs, path, filename, fmd, compression, partition_on)
+              for filename, part in zip(filenames, df.to_delayed())]
+
+    return delayed(_write_metadata)(writes, filenames, fmd, path, open_with, sep)
+
+
+def _write_metadata(writes, filenames, fmd, path, open_with, sep):
+    """ Write Parquet metadata after writing all row groups
+
+    See Also
+    --------
+    to_parquet
+    """
+    import fastparquet
+    fmd = copy.copy(fmd)
+    for fn, rg in zip(filenames, writes):
+        if rg is not None:
+            if isinstance(rg, list):
+                for r in rg:
+                    fmd.row_groups.append(r)
+            else:
+                for chunk in rg.columns:
+                    chunk.file_path = fn
+                fmd.row_groups.append(rg)
+
+    fn = sep.join([path, '_metadata'])
+    fastparquet.writer.write_common_metadata(fn, fmd, open_with=open_with,
+                                             no_row_groups=False)
+
+    fn = sep.join([path, '_common_metadata'])
+    fastparquet.writer.write_common_metadata(fn, fmd, open_with=open_with)
+
+
 # ----------------------------------------------------------------------
 # PyArrow interface
 
+
 def _read_pyarrow(fs, paths, file_opener, columns=None, filters=None,
                   categories=None, index=None):
-    api = pyarrow_parquet
+    import pyarrow.parquet as pq
 
     if filters is not None:
         raise NotImplementedError("Predicate pushdown not implemented")
@@ -172,22 +287,11 @@ def _read_pyarrow(fs, paths, file_opener, columns=None, filters=None,
     if isinstance(columns, tuple):
         columns = list(columns)
 
-    dataset = api.ParquetDataset(
-        path_or_paths=paths,
-        filesystem=fs
-    )
+    dataset = pq.ParquetDataset(paths, filesystem=fs)
     schema = dataset.schema.to_arrow_schema()
     has_pandas_metadata = schema.metadata is not None and b'pandas' in schema.metadata
     task_name = 'read-parquet-' + tokenize(dataset, columns)
 
-    if index is False:
-        index_col = None
-    elif index is None and has_pandas_metadata:
-        pandas_metadata = json.loads(schema.metadata[b'pandas'].decode('utf8'))
-        index_col = pandas_metadata.get('index_columns', None)
-    else:
-        index_col = index
-
     if columns is None:
         all_columns = schema.names
     else:
@@ -199,33 +303,43 @@ def _read_pyarrow(fs, paths, file_opener, columns=None, filters=None,
     else:
         out_type = DataFrame
 
-    if index_col and index_col not in all_columns:
-        if isinstance(index_col, list):
-            all_columns = all_columns + index_col
+    if index is False:
+        index_cols = []
+    elif index is None:
+        if has_pandas_metadata:
+            pandas_metadata = json.loads(schema.metadata[b'pandas'].decode('utf8'))
+            index_cols = pandas_metadata.get('index_columns', [])
         else:
-            all_columns.append(index_col)
+            index_cols = []
+    else:
+        index_cols = index if isinstance(index, list) else [index]
 
-    divisions = (None,) * (len(dataset.pieces) + 1)
+    if index_cols:
+        all_columns = list(unique(all_columns + index_cols))
 
     dtypes = _get_pyarrow_dtypes(schema)
 
-    meta = _meta_from_dtypes(all_columns, schema.names, dtypes)
-    if index_col:
-        meta = meta.set_index(index_col)
+    meta = _meta_from_dtypes(all_columns, schema.names, dtypes, index_cols)
 
     if out_type == Series:
         assert len(meta.columns) == 1
         meta = meta[meta.columns[0]]
 
-    task_plan = {
-        (task_name, i): (_read_arrow_parquet_piece,
-                         file_opener,
-                         piece, all_columns,
-                         index_col,
-                         out_type == Series,
-                         dataset.partitions)
-        for i, piece in enumerate(dataset.pieces)
-    }
+    if dataset.pieces:
+        divisions = (None,) * (len(dataset.pieces) + 1)
+        task_plan = {
+            (task_name, i): (_read_pyarrow_parquet_piece,
+                             file_opener,
+                             piece,
+                             all_columns,
+                             index_cols,
+                             out_type == Series,
+                             dataset.partitions)
+            for i, piece in enumerate(dataset.pieces)
+        }
+    else:
+        divisions = (None, None)
+        task_plan = {(task_name, 0): meta}
 
     return out_type(task_plan, task_name, meta, divisions)
 
@@ -240,16 +354,20 @@ def _get_pyarrow_dtypes(schema):
     return dtypes
 
 
-def _read_arrow_parquet_piece(open_file_func, piece, columns, index_col,
-                              is_series, partitions):
+def _read_pyarrow_parquet_piece(open_file_func, piece, columns, index_cols,
+                                is_series, partitions):
     with open_file_func(piece.path, mode='rb') as f:
         table = piece.read(columns=columns,  partitions=partitions,
                            use_pandas_metadata=True,
                            file=f)
     df = table.to_pandas()
-    if index_col is not None:
-        if not df.index.name == index_col[0]:
-            df = df.set_index(index_col)
+    if (index_cols and df.index.name is None and
+            len(df.columns.intersection(index_cols))):
+        # Index should be set, but it isn't
+        df = df.set_index(index_cols)
+    elif not index_cols and df.index.name is not None:
+        # Index shouldn't be set, but it is
+        df = df.reset_index(drop=False)
 
     if is_series:
         return df[df.columns[0]]
@@ -257,8 +375,116 @@ def _read_arrow_parquet_piece(open_file_func, piece, columns, index_col,
         return df
 
 
+def _write_pyarrow(df, path, write_index=None, append=False,
+                   ignore_divisions=False, partition_on=None,
+                   storage_options=None, **kwargs):
+    if append:
+        raise NotImplementedError("`append` not implemented for "
+                                  "`engine='pyarrow'`")
+
+    if partition_on:
+        raise NotImplementedError("`partition_on` not implemented for "
+                                  "`engine='pyarrow'`")
+
+    if write_index is None and df.known_divisions:
+        write_index = True
+
+    fs, paths, open_with = get_fs_paths_myopen(path, None, 'wb',
+                                               **(storage_options or {}))
+    fs.mkdirs(path)
+
+    template = fs.sep.join([path, 'part.%i.parquet'])
+
+    write = delayed(_write_partition_pyarrow)
+    first_kwargs = kwargs.copy()
+    first_kwargs['metadata_path'] = fs.sep.join([path, '_metadata'])
+    writes = [write(part, open_with, template % i, write_index,
+                    **(kwargs if i else first_kwargs))
+              for i, part in enumerate(df.to_delayed())]
+    return delayed(writes)
+
+
+def _write_partition_pyarrow(df, open_with, filename, write_index,
+                             metadata_path=None, **kwargs):
+    import pyarrow as pa
+    from pyarrow import parquet
+    t = pa.Table.from_pandas(df, preserve_index=write_index)
+
+    with open_with(filename, 'wb') as fil:
+        parquet.write_table(t, fil, **kwargs)
+
+    if metadata_path is not None:
+        with open_with(metadata_path, 'wb') as fil:
+            parquet.write_metadata(t.schema, fil, **kwargs)
+
+
 # ----------------------------------------------------------------------
-# User read API
+# User API
+
+
+_ENGINES = {}
+
+
+def get_engine(engine):
+    """Get the parquet engine backend implementation.
+
+    Parameters
+    ----------
+    engine : {'auto', 'fastparquet', 'pyarrow'}, default 'auto'
+        Parquet reader library to use. Default is first installed in this list.
+
+    Returns
+    -------
+    A dict containing a ``'read'`` and ``'write'`` function.
+    """
+    if engine in _ENGINES:
+        return _ENGINES[engine]
+
+    if engine == 'auto':
+        for eng in ['fastparquet', 'pyarrow']:
+            try:
+                return get_engine(eng)
+            except ImportError:
+                pass
+        else:
+            raise ImportError("Please install either fastparquet or pyarrow")
+
+    elif engine == 'fastparquet':
+        try:
+            import fastparquet
+        except ImportError:
+            raise ImportError("fastparquet not installed")
+
+        @normalize_token.register(fastparquet.ParquetFile)
+        def normalize_ParquetFile(pf):
+            return (type(pf), pf.fn, pf.sep) + normalize_token(pf.open)
+
+        _ENGINES['fastparquet'] = eng = {'read': _read_fastparquet,
+                                         'write': _write_fastparquet}
+        return eng
+
+    elif engine == 'pyarrow':
+        try:
+            import pyarrow.parquet as pq
+        except ImportError:
+            raise ImportError("pyarrow not installed")
+
+        @normalize_token.register(pq.ParquetDataset)
+        def normalize_PyArrowParquetDataset(ds):
+            return (type(ds), ds.paths)
+
+        _ENGINES['pyarrow'] = eng = {'read': _read_pyarrow,
+                                     'write': _write_pyarrow}
+        return eng
+
+    elif engine == 'arrow':
+        warnings.warn("parquet with `engine='arrow'` is deprecated, "
+                      "use `engine='pyarrow'` instead")
+        return get_engine('pyarrow')
+
+    else:
+        raise ValueError('Unsupported engine type: {0}'.format(engine))
+
 
 def read_parquet(path, columns=None, filters=None, categories=None, index=None,
                  storage_options=None, engine='auto'):
@@ -292,7 +518,7 @@ def read_parquet(path, columns=None, filters=None, categories=None, index=None,
         data written by dask/fastparquet, not otherwise.
     storage_options : dict
         Key/value pairs to be passed on to the file-system backend, if any.
-    engine : {'auto', 'fastparquet', 'arrow'}, default 'auto'
+    engine : {'auto', 'fastparquet', 'pyarrow'}, default 'auto'
         Parquet reader library to use. If only one library is installed, it
         will use that one; if both, it will use 'fastparquet'
 
@@ -307,36 +533,15 @@ def read_parquet(path, columns=None, filters=None, categories=None, index=None,
     fs, paths, file_opener = get_fs_paths_myopen(path, None, 'rb',
                                                  **(storage_options or {}))
 
-    if engine == 'auto':
-        if fastparquet:
-            engine = 'fastparquet'
-        elif pyarrow_parquet:
-            engine = 'arrow'
-        else:
-            raise ImportError("Please install either fastparquet or pyarrow")
-    elif engine == 'fastparquet':
-        if not fastparquet:
-            raise ImportError("fastparquet not installed")
-    elif engine == 'arrow':
-        if not pyarrow_parquet:
-            raise ImportError("pyarrow not installed")
-    else:
-        raise ValueError('Unsupported engine type: {0}'.format(engine))
+    read = get_engine(engine)['read']
 
-    if engine == 'fastparquet':
-        return _read_fastparquet(fs, paths, file_opener, columns=columns,
-                                 filters=filters,
-                                 categories=categories, index=index)
-    else:
-        return _read_pyarrow(fs, paths, file_opener, columns=columns,
-                             filters=filters,
-                             categories=categories, index=index)
+    return read(fs, paths, file_opener, columns=columns, filters=filters,
+                categories=categories, index=index)
 
 
-def to_parquet(df, path, compression=None, write_index=None, has_nulls=True,
-               fixed_text=None, object_encoding=None, storage_options=None,
+def to_parquet(df, path, engine='auto', compression='default', write_index=None,
                append=False, ignore_divisions=False, partition_on=None,
-               compute=True, times='int64'):
+               storage_options=None, compute=True, **kwargs):
     """Store Dask.dataframe to Parquet files
 
     Notes
@@ -345,180 +550,74 @@ def to_parquet(df, path, compression=None, write_index=None, has_nulls=True,
 
     Parameters
     ----------
-    df : Dask.dataframe
+    df : dask.dataframe.DataFrame
     path : string
         Destination directory for data.  Prepend with protocol like ``s3://``
         or ``hdfs://`` for remote data.
-    compression : string or dict
-        Either a string like "SNAPPY" or a dictionary mapping column names to
-        compressors like ``{"name": "GZIP", "values": "SNAPPY"}``
-    write_index : boolean
-        Whether or not to write the index.  Defaults to True *if* divisions are
+    engine : {'auto', 'fastparquet', 'pyarrow'}, default 'auto'
+        Parquet library to use. If only one library is installed, it will use
+        that one; if both, it will use 'fastparquet'.
+    compression : string or dict, optional
+        Either a string like ``"snappy"`` or a dictionary mapping column names
+        to compressors like ``{"name": "gzip", "values": "snappy"}``. The
+        default is ``"default"``, which uses the default compression for
+        whichever engine is selected.
+    write_index : boolean, optional
+        Whether or not to write the index. Defaults to True *if* divisions are
         known.
-    has_nulls : bool, list or 'infer'
-        Specifies whether to write NULLs information for columns. If bools,
-        apply to all columns, if list, use for only the named columns, if
-        'infer', use only for columns which don't have a sentinel NULL marker
-        (currently object columns only).
-    fixed_text : dict {col: int}
-        For column types that are written as bytes (bytes, utf8 strings, or
-        json and bson-encoded objects), if a column is included here, the
-        data will be written in fixed-length format, which should be faster
-        but can potentially result in truncation.
-    object_encoding : dict {col: bytes|utf8|json|bson} or str
-        For object columns, specify how to encode to bytes. If a str, same
-        encoding is applied to all object columns.
-    storage_options : dict
-        Key/value pairs to be passed on to the file-system backend, if any.
-    append: bool (False)
-        If False, construct data-set from scratch; if True, add new
-        row-group(s) to existing data-set. In the latter case, the data-set
+    append : bool, optional
+        If False (default), construct data-set from scratch. If True, add new
+        row-group(s) to an existing data-set. In the latter case, the data-set
         must exist, and the schema must match the input data.
-    ignore_divisions: bool (False)
-        If False raises error when previous divisions overlap with the new
-        appended divisions. Ignored if append=False.
-    partition_on: list
+    ignore_divisions : bool, optional
+        If False (default) raises error when previous divisions overlap with
+        the new appended divisions. Ignored if append=False.
+    partition_on : list, optional
         Construct directory-based partitioning by splitting on these fields'
         values. Each dask partition will result in one or more datafiles,
         there will be no global groupby.
-    times: 'int64' (default), or 'int96':
-        In "int64" mode, datetimes are written as 8-byte integers, us
-        resolution; in "int96" mode, they are written as 12-byte blocks, with
-        the first 8 bytes as ns within the day, the next 4 bytes the julian day.
-        'int96' mode is included only for compatibility.
-    compute: bool (True)
-        If true (default) then we compute immediately.
-        If False then we return a dask.delayed object for future computation.
-
-    This uses the fastparquet project:
-    http://fastparquet.readthedocs.io/en/latest
+    storage_options : dict, optional
+        Key/value pairs to be passed on to the file-system backend, if any.
+    compute : bool, optional
+        If True (default) then the result is computed immediately. If False
+        then a ``dask.delayed`` object is returned for future computation.
+    **kwargs
+        Extra options to be passed on to the specific backend.
 
     Examples
     --------
     >>> df = dd.read_csv(...)  # doctest: +SKIP
-    >>> to_parquet('/path/to/output/', df, compression='SNAPPY')  # doctest: +SKIP
+    >>> to_parquet('/path/to/output/', df, compression='snappy')  # doctest: +SKIP
 
     See Also
     --------
     read_parquet: Read parquet data to dask.dataframe
     """
-    import fastparquet
-    partition_on = partition_on or []
-
     # TODO: remove once deprecation cycle is finished
     if isinstance(path, DataFrame):
         warnings.warn("DeprecationWarning: The order of `df` and `path` in "
                       "`dd.to_parquet` has switched, please update your code")
         df, path = path, df
 
-    fs, paths, myopen = get_fs_paths_myopen(path, None, 'wb',
-                                            **(storage_options or {}))
-    fs.mkdirs(path)
-    sep = fs.sep
-
-    if write_index is True or write_index is None and df.known_divisions:
-        new_divisions = df.divisions
-        df = df.reset_index()
-        index_col = df.columns[0]
-    else:
-        ignore_divisions = True
-        index_col = None
+    partition_on = partition_on or []
 
-    object_encoding = object_encoding or 'utf8'
-    if object_encoding == 'infer' or (isinstance(object_encoding, dict) and
-                                      'infer' in object_encoding.values()):
-        raise ValueError('"infer" not allowed as object encoding, '
-                         'because this required data in memory.')
     if set(partition_on) - set(df.columns):
         raise ValueError('Partitioning on non-existent column')
-    fmd = fastparquet.writer.make_metadata(df._meta,
-                                           has_nulls=has_nulls,
-                                           fixed_text=fixed_text,
-                                           object_encoding=object_encoding,
-                                           index_cols=[index_col],
-                                           ignore_columns=partition_on,
-                                           times=times)
-
-    if append:
-        pf = fastparquet.api.ParquetFile(path, open_with=myopen, sep=sep)
-        if pf.file_scheme not in ['hive', 'empty', 'flat']:
-            raise ValueError('Requested file scheme is hive, '
-                             'but existing file scheme is not.')
-        elif ((set(pf.columns) != set(df.columns) - set(partition_on)) or
-              (set(partition_on) != set(pf.cats))):
-            raise ValueError('Appended columns not the same.\n'
-                             'New: {} | Previous: {}'
-                             .format(pf.columns, list(df.columns)))
-        elif set(pf.dtypes[c] for c in pf.columns) != set(df[pf.columns].dtypes):
-            raise ValueError('Appended dtypes differ.\n{}'
-                             .format(set(pf.dtypes.items()) ^
-                                     set(df.dtypes.iteritems())))
-        else:
-            df = df[pf.columns + partition_on]
-
-        fmd = pf.fmd
-        i_offset = fastparquet.writer.find_max_part(fmd.row_groups)
 
-        if not ignore_divisions:
-            minmax = fastparquet.api.sorted_partitioned_columns(pf)
-            if index_col in minmax:
+    if compression != 'default':
+        kwargs['compression'] = compression
 
-                divisions = list(minmax[index_col]['min']) + [
-                    minmax[index_col]['max'][-1]]
+    write = get_engine(engine)['write']
 
-                if new_divisions[0] < divisions[-1]:
-                    raise ValueError(
-                        'Appended divisions overlapping with the previous ones.'
-                        '\nNew: {} | Previous: {}'
-                        .format(divisions[-1], new_divisions[0]))
-    else:
-        i_offset = 0
+    out = write(df, path, write_index=write_index, append=append,
+                ignore_divisions=ignore_divisions, partition_on=partition_on,
+                storage_options=storage_options, **kwargs)
 
-    filenames = ['part.%i.parquet' % (i + i_offset)
-                 for i in range(df.npartitions)]
-
-    if partition_on:
-        write = delayed(fastparquet.writer.partition_on_columns)
-        writes = [write(partition, partition_on, path, filename, fmd, sep,
-                        compression, fs.open, fs.mkdirs)
-                  for filename, partition in zip(filenames, df.to_delayed())]
-    else:
-        write = delayed(fastparquet.writer.make_part_file)
-        writes = [write(myopen(sep.join([path, filename]), 'wb'), partition,
-                        fmd.schema, compression=compression)
-                  for filename, partition in zip(filenames, df.to_delayed())]
-
-    out = delayed(_write_metadata)(writes, filenames, fmd, path, myopen, sep)
     if compute:
         out.compute()
         return None
     return out
 
 
-def _write_metadata(writes, filenames, fmd, path, myopen, sep):
-    """ Write Parquet metadata after writing all row groups
-
-    See Also
-    --------
-    to_parquet
-    """
-    for fn, rg in zip(filenames, writes):
-        if rg is not None:
-            if isinstance(rg, list):
-                for r in rg:
-                    fmd.row_groups.append(r)
-            else:
-                for chunk in rg.columns:
-                    chunk.file_path = fn
-                fmd.row_groups.append(rg)
-
-    fn = sep.join([path, '_metadata'])
-    fastparquet.writer.write_common_metadata(fn, fmd, open_with=myopen,
-                                             no_row_groups=False)
-
-    fn = sep.join([path, '_common_metadata'])
-    fastparquet.writer.write_common_metadata(fn, fmd, open_with=myopen)
-
-
 if PY3:
     DataFrame.to_parquet.__doc__ = to_parquet.__doc__


Test patch:
diff --git a/dask/bytes/tests/test_s3.py b/dask/bytes/tests/test_s3.py
--- a/dask/bytes/tests/test_s3.py
+++ b/dask/bytes/tests/test_s3.py
@@ -282,12 +282,10 @@ def test_read_text_passes_through_options():
         assert df.count().compute(get=get) == 3
 
 
-@pytest.mark.parametrize("engine", ['arrow', 'fastparquet'])
+@pytest.mark.parametrize("engine", ['pyarrow', 'fastparquet'])
 def test_parquet(s3, engine):
     dd = pytest.importorskip('dask.dataframe')
-    pytest.importorskip('fastparquet')
-    if engine == 'arrow':
-        pytest.importorskip('pyarrow')
+    pytest.importorskip(engine)
     import pandas as pd
     import numpy as np
 
@@ -301,7 +299,7 @@ def test_parquet(s3, engine):
                              size=1000).astype("O")},
                         index=pd.Index(np.arange(1000), name='foo'))
     df = dd.from_pandas(data, chunksize=500)
-    df.to_parquet(url, object_encoding='utf8')
+    df.to_parquet(url, engine=engine)
 
     files = [f.split('/')[-1] for f in s3.ls(url)]
     assert '_metadata' in files
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -2,6 +2,7 @@
                         unicode_literals)
 
 import os
+
 import numpy as np
 import pandas as pd
 import pandas.util.testing as tm
@@ -9,128 +10,172 @@
 
 import dask
 import dask.multiprocessing
-from dask.utils import tmpdir, tmpfile
 import dask.dataframe as dd
 from dask.dataframe.utils import assert_eq
 
-fastparquet = pytest.importorskip('fastparquet')
+try:
+    import fastparquet
+except ImportError:
+    fastparquet = False
 
 try:
-    import pyarrow.parquet as pyarrow  # noqa
+    import pyarrow.parquet as pq
 except ImportError:
-    pyarrow = False
+    pq = False
 
 
 df = pd.DataFrame({'x': [6, 2, 3, 4, 5],
                    'y': [1.0, 2.0, 1.0, 2.0, 1.0]},
                   index=pd.Index([10, 20, 30, 40, 50], name='myindex'))
 
+ddf = dd.from_pandas(df, npartitions=3)
 
-@pytest.fixture
-def fn(tmpdir):
-    ddf = dd.from_pandas(df, npartitions=3)
-    dd.to_parquet(ddf, str(tmpdir))
-
-    return str(tmpdir)
 
-
-@pytest.fixture(params=[
-    pytest.mark.skipif(not fastparquet, 'fastparquet',
-                       reason='fastparquet not found'),
-    pytest.mark.skipif(not pyarrow, 'arrow',
-                       reason='pyarrow not found')
-])
+@pytest.fixture(params=[pytest.mark.skipif(not fastparquet, 'fastparquet',
+                                           reason='fastparquet not found'),
+                        pytest.mark.skipif(not pq, 'pyarrow',
+                                           reason='pyarrow not found')])
 def engine(request):
     return request.param
 
 
-def test_local(engine):
-    with tmpdir() as tmp:
-        tmp = str(tmp)
-        data = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
-                             'i64': np.arange(1000, dtype=np.int64),
-                             'f': np.arange(1000, dtype=np.float64),
-                             'bhello': np.random.choice(['hello', 'yo', 'people'], size=1000).astype("O")})
-        df = dd.from_pandas(data, chunksize=500)
+def check_fastparquet():
+    if not fastparquet:
+        pytest.skip('fastparquet not found')
 
-        df.to_parquet(tmp, write_index=False, object_encoding='utf8')
 
-        files = os.listdir(tmp)
-        assert '_metadata' in files
-        assert 'part.0.parquet' in files
+def check_pyarrow():
+    if not pq:
+        pytest.skip('pyarrow not found')
 
-        df2 = dd.read_parquet(tmp, index=False, engine=engine)
 
-        assert len(df2.divisions) > 1
+def write_read_engines(xfail_arrow_to_fastparquet=True):
+    if xfail_arrow_to_fastparquet:
+        xfail = (pytest.mark.xfail(reason="Can't read arrow directories with fastparquet"),)
+    else:
+        xfail = ()
+    ff = () if fastparquet else (pytest.mark.skip(reason='fastparquet not found'),)
+    aa = () if pq else (pytest.mark.skip(reason='pyarrow not found'),)
+    engines = [pytest.param('fastparquet', 'fastparquet', marks=ff),
+               pytest.param('pyarrow', 'pyarrow', marks=aa),
+               pytest.param('fastparquet', 'pyarrow', marks=ff + aa),
+               pytest.param('pyarrow', 'fastparquet', marks=ff + aa + xfail)]
+    return pytest.mark.parametrize(('write_engine', 'read_engine'), engines)
 
-        out = df2.compute(get=dask.get).reset_index()
 
-        for column in df.columns:
-            assert (data[column] == out[column]).all()
+write_read_engines_xfail = write_read_engines(xfail_arrow_to_fastparquet=True)
 
 
-def test_index(fn, engine):
-    ddf = dd.read_parquet(fn, engine=engine)
-    assert_eq(df, ddf)
+@write_read_engines_xfail
+def test_local(tmpdir, write_engine, read_engine):
+    tmp = str(tmpdir)
+    data = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
+                         'i64': np.arange(1000, dtype=np.int64),
+                         'f': np.arange(1000, dtype=np.float64),
+                         'bhello': np.random.choice(['hello', 'yo', 'people'], size=1000).astype("O")})
+    df = dd.from_pandas(data, chunksize=500)
+
+    df.to_parquet(tmp, write_index=False, engine=write_engine)
+
+    files = os.listdir(tmp)
+    assert '_metadata' in files
+    assert 'part.0.parquet' in files
+
+    df2 = dd.read_parquet(tmp, index=False, engine=read_engine)
+
+    assert len(df2.divisions) > 1
+
+    out = df2.compute(get=dask.get).reset_index()
 
+    for column in df.columns:
+        assert (data[column] == out[column]).all()
 
-def test_empty_index(engine):
-    with tmpdir() as tmp:
-        df = pd.DataFrame(columns=['a', 'b'])
+
+@write_read_engines_xfail
+def test_index(tmpdir, write_engine, read_engine):
+    fn = str(tmpdir)
+    ddf.to_parquet(fn, engine=write_engine)
+    ddf2 = dd.read_parquet(fn, engine=read_engine)
+    assert_eq(df, ddf2)
+
+
+@pytest.mark.parametrize('index', [False, True])
+@write_read_engines_xfail
+def test_empty(tmpdir, write_engine, read_engine, index):
+    fn = str(tmpdir)
+    df = pd.DataFrame({'a': ['a', 'b', 'b'], 'b': [4, 5, 6]})[:0]
+    if index:
         df.set_index('a', inplace=True, drop=True)
-        ddf = dd.from_pandas(df, npartitions=2)
-        ddf.to_parquet(tmp, write_index=True)
-        read_df = dd.read_parquet(tmp, engine=engine)
-        assert read_df.index.name == 'a'
+    ddf = dd.from_pandas(df, npartitions=2)
 
+    ddf.to_parquet(fn, write_index=index, engine=write_engine)
+    read_df = dd.read_parquet(fn, engine=read_engine)
+    assert_eq(df, read_df)
 
-def test_glob(fn, engine):
+
+@write_read_engines(xfail_arrow_to_fastparquet=False)
+def test_read_glob(tmpdir, write_engine, read_engine):
+    fn = str(tmpdir)
+    ddf.to_parquet(fn, engine=write_engine)
     os.unlink(os.path.join(fn, '_metadata'))
+
     files = os.listdir(fn)
     assert '_metadata' not in files
-    ddf = dd.read_parquet(os.path.join(fn, '*'), engine=engine)
-    assert_eq(df, ddf)
-
 
-def test_auto_add_index(fn, engine):
-    ddf = dd.read_parquet(fn, columns=['x'], index='myindex', engine=engine)
-    assert_eq(df[['x']], ddf)
+    ddf2 = dd.read_parquet(os.path.join(fn, '*'), engine=read_engine)
+    assert_eq(df, ddf2)
 
 
-def test_index_column(fn, engine):
-    ddf = dd.read_parquet(fn, columns=['myindex'], index='myindex', engine=engine)
-    assert_eq(df[[]], ddf)
+@write_read_engines_xfail
+def test_auto_add_index(tmpdir, write_engine, read_engine):
+    fn = str(tmpdir)
+    ddf.to_parquet(fn, engine=write_engine)
+    ddf2 = dd.read_parquet(fn, columns=['x'], index='myindex', engine=read_engine)
+    assert_eq(df[['x']], ddf2)
 
 
-def test_index_column_no_index(fn, engine):
-    ddf = dd.read_parquet(fn, columns=['myindex'])
-    assert_eq(df[[]], ddf)
+@write_read_engines_xfail
+def test_index_column_false_index(tmpdir, write_engine, read_engine):
+    fn = str(tmpdir)
+    ddf.to_parquet(fn, engine=write_engine)
+    ddf2 = dd.read_parquet(fn, columns=['myindex'], index=False, engine=read_engine)
+    assert_eq(pd.DataFrame(df.index), ddf2, check_index=False)
 
 
-def test_index_column_false_index(fn, engine):
-    ddf = dd.read_parquet(fn, columns=['myindex'], index=False, engine=engine)
-    assert_eq(pd.DataFrame(df.index), ddf, check_index=False)
+@pytest.mark.parametrize("columns", [['myindex'], []])
+@pytest.mark.parametrize("index", ['myindex', None])
+@write_read_engines_xfail
+def test_columns_index(tmpdir, write_engine, read_engine, columns, index):
+    fn = str(tmpdir)
+    ddf.to_parquet(fn, engine=write_engine)
+    ddf2 = dd.read_parquet(fn, columns=columns, index=index, engine=read_engine)
+    assert_eq(df[[]], ddf2)
 
 
-def test_no_columns_yes_index(fn, engine):
-    ddf = dd.read_parquet(fn, columns=[], index='myindex', engine=engine)
-    assert_eq(df[[]], ddf)
-
+@write_read_engines_xfail
+def test_no_index(tmpdir, write_engine, read_engine):
+    fn = str(tmpdir)
+    df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
+    ddf = dd.from_pandas(df, npartitions=2)
+    ddf.to_parquet(fn, write_index=False, engine=write_engine)
+    ddf2 = dd.read_parquet(fn, engine=read_engine)
+    assert_eq(df, ddf2, check_index=False)
 
-def test_no_columns_no_index(fn, engine):
-    ddf = dd.read_parquet(fn, columns=[], engine=engine)
-    assert_eq(df[[]], ddf)
 
+def test_read_series(tmpdir, engine):
+    fn = str(tmpdir)
+    ddf.to_parquet(fn, engine=engine)
+    ddf2 = dd.read_parquet(fn, columns=['x'], engine=engine)
+    assert_eq(df[['x']], ddf2)
 
-def test_series(fn, engine):
-    ddf = dd.read_parquet(fn, columns=['x'], engine=engine)
-    assert_eq(df[['x']], ddf)
+    ddf2 = dd.read_parquet(fn, columns='x', index='myindex', engine=engine)
+    assert_eq(df.x, ddf2)
 
-    ddf = dd.read_parquet(fn, columns='x', index='myindex', engine=engine)
-    assert_eq(df.x, ddf)
 
+def test_names(tmpdir, engine):
+    fn = str(tmpdir)
+    ddf.to_parquet(fn, engine=engine)
 
-def test_names(fn, engine):
     def read(fn, **kwargs):
         return dd.read_parquet(fn, engine=engine, **kwargs)
 
@@ -144,190 +189,198 @@ def read(fn, **kwargs):
 
 
 @pytest.mark.parametrize('c', [['x'], 'x', ['x', 'y'], []])
-def test_optimize(fn, c):
-    ddf = dd.read_parquet(fn)
-    assert_eq(df[c], ddf[c])
-    x = ddf[c]
+def test_optimize(tmpdir, c):
+    check_fastparquet()
+    fn = str(tmpdir)
+    ddf.to_parquet(fn)
+    ddf2 = dd.read_parquet(fn)
+    assert_eq(df[c], ddf2[c])
+    x = ddf2[c]
 
     dsk = x.__dask_optimize__(x.dask, x.__dask_keys__())
     assert len(dsk) == x.npartitions
     assert all(v[4] == c for v in dsk.values())
 
 
-def test_roundtrip_from_pandas(engine):
-    with tmpfile() as fn:
-        df = pd.DataFrame({'x': [1, 2, 3]})
-        fastparquet.write(fn, df)
-        ddf = dd.io.parquet.read_parquet(fn, index=False, engine=engine)
-        assert_eq(df, ddf)
+@pytest.mark.skipif(not hasattr(pd.DataFrame, 'to_parquet'),
+                    reason="no to_parquet method")
+@write_read_engines(False)
+def test_roundtrip_from_pandas(tmpdir, write_engine, read_engine):
+    fn = str(tmpdir.join('test.parquet'))
+    df = pd.DataFrame({'x': [1, 2, 3]})
+    df.to_parquet(fn, engine=write_engine)
+    ddf = dd.read_parquet(fn, engine=read_engine)
+    assert_eq(df, ddf)
 
 
-def test_categorical():
-    with tmpdir() as tmp:
-        df = pd.DataFrame({'x': ['a', 'b', 'c'] * 100},
-                          dtype='category')
-        ddf = dd.from_pandas(df, npartitions=3)
-        dd.to_parquet(ddf, tmp)
+def test_categorical(tmpdir):
+    check_fastparquet()
+    tmp = str(tmpdir)
+    df = pd.DataFrame({'x': ['a', 'b', 'c'] * 100}, dtype='category')
+    ddf = dd.from_pandas(df, npartitions=3)
+    dd.to_parquet(ddf, tmp)
 
-        ddf2 = dd.read_parquet(tmp, categories=['x'])
-        assert ddf2.compute().x.cat.categories.tolist() == ['a', 'b', 'c']
+    ddf2 = dd.read_parquet(tmp, categories=['x'])
+    assert ddf2.compute().x.cat.categories.tolist() == ['a', 'b', 'c']
 
-        # autocat
-        ddf2 = dd.read_parquet(tmp)
-        assert ddf2.compute().x.cat.categories.tolist() == ['a', 'b', 'c']
+    # autocat
+    ddf2 = dd.read_parquet(tmp)
+    assert ddf2.compute().x.cat.categories.tolist() == ['a', 'b', 'c']
 
-        ddf2.loc[:1000].compute()
-        df.index.name = 'index'  # defaults to 'index' in this case
-        assert assert_eq(df, ddf2)
+    ddf2.loc[:1000].compute()
+    df.index.name = 'index'  # defaults to 'index' in this case
+    assert assert_eq(df, ddf2)
 
-        # dereference cats
-        ddf2 = dd.read_parquet(tmp, categories=[])
+    # dereference cats
+    ddf2 = dd.read_parquet(tmp, categories=[])
 
-        ddf2.loc[:1000].compute()
-        assert (df.x == ddf2.x).all()
+    ddf2.loc[:1000].compute()
+    assert (df.x == ddf2.x).all()
 
 
-def test_append(engine):
+def test_append(tmpdir, engine):
     """Test that appended parquet equal to the original one."""
-    with tmpdir() as tmp:
-        df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
-                           'i64': np.arange(1000, dtype=np.int64),
-                           'f': np.arange(1000, dtype=np.float64),
-                           'bhello': np.random.choice(['hello', 'yo', 'people'],
-                                                      size=1000).astype("O")})
-        df.index.name = 'index'
-
-        half = len(df) // 2
-        ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
-        ddf2 = dd.from_pandas(df.iloc[half:], chunksize=100)
-        ddf1.to_parquet(tmp)
-        ddf2.to_parquet(tmp, append=True)
+    check_fastparquet()
+    tmp = str(tmpdir)
+    df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
+                       'i64': np.arange(1000, dtype=np.int64),
+                       'f': np.arange(1000, dtype=np.float64),
+                       'bhello': np.random.choice(['hello', 'yo', 'people'],
+                                                  size=1000).astype("O")})
+    df.index.name = 'index'
 
-        ddf3 = dd.read_parquet(tmp, engine=engine)
-        assert_eq(df, ddf3)
-
-
-def test_append_with_partition():
-    with tmpdir() as tmp:
-        df0 = pd.DataFrame({'lat': np.arange(0, 10), 'lon': np.arange(10, 20),
-                            'value': np.arange(100, 110)})
-        df0.index.name = 'index'
-        df1 = pd.DataFrame({'lat': np.arange(10, 20), 'lon': np.arange(10, 20),
-                            'value': np.arange(120, 130)})
-        df1.index.name = 'index'
-        dd_df0 = dd.from_pandas(df0, npartitions=1)
-        dd_df1 = dd.from_pandas(df1, npartitions=1)
-        dd.to_parquet(dd_df0, tmp, partition_on=['lon'])
-        dd.to_parquet(dd_df1, tmp, partition_on=['lon'], append=True,
-                      ignore_divisions=True)
-
-        out = dd.read_parquet(tmp).compute()
+    half = len(df) // 2
+    ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
+    ddf2 = dd.from_pandas(df.iloc[half:], chunksize=100)
+    ddf1.to_parquet(tmp)
+    ddf2.to_parquet(tmp, append=True)
+
+    ddf3 = dd.read_parquet(tmp, engine=engine)
+    assert_eq(df, ddf3)
+
+
+def test_append_with_partition(tmpdir):
+    check_fastparquet()
+    tmp = str(tmpdir)
+    df0 = pd.DataFrame({'lat': np.arange(0, 10), 'lon': np.arange(10, 20),
+                        'value': np.arange(100, 110)})
+    df0.index.name = 'index'
+    df1 = pd.DataFrame({'lat': np.arange(10, 20), 'lon': np.arange(10, 20),
+                        'value': np.arange(120, 130)})
+    df1.index.name = 'index'
+    dd_df0 = dd.from_pandas(df0, npartitions=1)
+    dd_df1 = dd.from_pandas(df1, npartitions=1)
+    dd.to_parquet(dd_df0, tmp, partition_on=['lon'])
+    dd.to_parquet(dd_df1, tmp, partition_on=['lon'], append=True,
+                  ignore_divisions=True)
+
+    out = dd.read_parquet(tmp).compute()
     out['lon'] = out.lon.astype('int64')  # just to pass assert
     # sort required since partitioning breaks index order
     assert_eq(out.sort_values('value'), pd.concat([df0, df1])[out.columns],
               check_index=False)
 
 
-def test_append_wo_index():
+def test_append_wo_index(tmpdir):
     """Test append with write_index=False."""
-    with tmpdir() as tmp:
-        df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
-                           'i64': np.arange(1000, dtype=np.int64),
-                           'f': np.arange(1000, dtype=np.float64),
-                           'bhello': np.random.choice(['hello', 'yo', 'people'],
-                                                      size=1000).astype("O")})
-        half = len(df) // 2
-        ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
-        ddf2 = dd.from_pandas(df.iloc[half:], chunksize=100)
-        ddf1.to_parquet(tmp)
-        with pytest.raises(ValueError) as excinfo:
-            ddf2.to_parquet(tmp, write_index=False, append=True)
-
-        assert 'Appended columns' in str(excinfo.value)
-
-    with tmpdir() as tmp:
-        ddf1.to_parquet(tmp, write_index=False)
+    check_fastparquet()
+    tmp = str(tmpdir.join('tmp1.parquet'))
+    df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
+                       'i64': np.arange(1000, dtype=np.int64),
+                       'f': np.arange(1000, dtype=np.float64),
+                       'bhello': np.random.choice(['hello', 'yo', 'people'],
+                                                  size=1000).astype("O")})
+    half = len(df) // 2
+    ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
+    ddf2 = dd.from_pandas(df.iloc[half:], chunksize=100)
+    ddf1.to_parquet(tmp)
+
+    with pytest.raises(ValueError) as excinfo:
         ddf2.to_parquet(tmp, write_index=False, append=True)
+    assert 'Appended columns' in str(excinfo.value)
 
-        ddf3 = dd.read_parquet(tmp, index='f')
-        assert_eq(df.set_index('f'), ddf3)
-
+    tmp = str(tmpdir.join('tmp2.parquet'))
+    ddf1.to_parquet(tmp, write_index=False)
+    ddf2.to_parquet(tmp, write_index=False, append=True)
 
-def test_append_overlapping_divisions():
-    """Test raising of error when divisions overlapping."""
-    with tmpdir() as tmp:
-        df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
-                           'i64': np.arange(1000, dtype=np.int64),
-                           'f': np.arange(1000, dtype=np.float64),
-                           'bhello': np.random.choice(
-                               ['hello', 'yo', 'people'],
-                               size=1000).astype("O")})
-        half = len(df) // 2
-        ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
-        ddf2 = dd.from_pandas(df.iloc[half - 10:], chunksize=100)
-        ddf1.to_parquet(tmp)
+    ddf3 = dd.read_parquet(tmp, index='f')
+    assert_eq(df.set_index('f'), ddf3)
 
-        with pytest.raises(ValueError) as excinfo:
-            ddf2.to_parquet(tmp, append=True)
 
-        assert 'Appended divisions' in str(excinfo.value)
+def test_append_overlapping_divisions(tmpdir):
+    """Test raising of error when divisions overlapping."""
+    check_fastparquet()
+    tmp = str(tmpdir)
+    df = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
+                       'i64': np.arange(1000, dtype=np.int64),
+                       'f': np.arange(1000, dtype=np.float64),
+                       'bhello': np.random.choice(['hello', 'yo', 'people'],
+                                                  size=1000).astype("O")})
+    half = len(df) // 2
+    ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
+    ddf2 = dd.from_pandas(df.iloc[half - 10:], chunksize=100)
+    ddf1.to_parquet(tmp)
+
+    with pytest.raises(ValueError) as excinfo:
+        ddf2.to_parquet(tmp, append=True)
+    assert 'Appended divisions' in str(excinfo.value)
 
-        ddf2.to_parquet(tmp, append=True, ignore_divisions=True)
+    ddf2.to_parquet(tmp, append=True, ignore_divisions=True)
 
 
-def test_append_different_columns():
+def test_append_different_columns(tmpdir):
     """Test raising of error when non equal columns."""
-    with tmpdir() as tmp:
-        df1 = pd.DataFrame({'i32': np.arange(100, dtype=np.int32)})
-        df2 = pd.DataFrame({'i64': np.arange(100, dtype=np.int64)})
-        df3 = pd.DataFrame({'i32': np.arange(100, dtype=np.int64)})
+    check_fastparquet()
+    tmp = str(tmpdir)
+    df1 = pd.DataFrame({'i32': np.arange(100, dtype=np.int32)})
+    df2 = pd.DataFrame({'i64': np.arange(100, dtype=np.int64)})
+    df3 = pd.DataFrame({'i32': np.arange(100, dtype=np.int64)})
 
-        ddf1 = dd.from_pandas(df1, chunksize=2)
-        ddf2 = dd.from_pandas(df2, chunksize=2)
-        ddf3 = dd.from_pandas(df3, chunksize=2)
+    ddf1 = dd.from_pandas(df1, chunksize=2)
+    ddf2 = dd.from_pandas(df2, chunksize=2)
+    ddf3 = dd.from_pandas(df3, chunksize=2)
 
-        ddf1.to_parquet(tmp)
+    ddf1.to_parquet(tmp)
 
-        with pytest.raises(ValueError) as excinfo:
-            ddf2.to_parquet(tmp, append=True)
-        assert 'Appended columns' in str(excinfo.value)
+    with pytest.raises(ValueError) as excinfo:
+        ddf2.to_parquet(tmp, append=True)
+    assert 'Appended columns' in str(excinfo.value)
 
-        with pytest.raises(ValueError) as excinfo:
-            ddf3.to_parquet(tmp, append=True)
-        assert 'Appended dtypes' in str(excinfo.value)
+    with pytest.raises(ValueError) as excinfo:
+        ddf3.to_parquet(tmp, append=True)
+    assert 'Appended dtypes' in str(excinfo.value)
 
 
-def test_ordering():
-    with tmpdir() as tmp:
-        tmp = str(tmp)
-        df = pd.DataFrame({'a': [1, 2, 3],
-                           'b': [10, 20, 30],
-                           'c': [100, 200, 300]},
-                          index=pd.Index([-1, -2, -3], name='myindex'),
-                          columns=['c', 'a', 'b'])
-        ddf = dd.from_pandas(df, npartitions=2)
-        dd.to_parquet(ddf, tmp)
+def test_ordering(tmpdir):
+    check_fastparquet()
+    tmp = str(tmpdir)
+    df = pd.DataFrame({'a': [1, 2, 3],
+                       'b': [10, 20, 30],
+                       'c': [100, 200, 300]},
+                      index=pd.Index([-1, -2, -3], name='myindex'),
+                      columns=['c', 'a', 'b'])
+    ddf = dd.from_pandas(df, npartitions=2)
+    dd.to_parquet(ddf, tmp)
 
-        pf = fastparquet.ParquetFile(tmp)
-        assert pf.columns == ['myindex', 'c', 'a', 'b']
+    pf = fastparquet.ParquetFile(tmp)
+    assert pf.columns == ['myindex', 'c', 'a', 'b']
 
-        ddf2 = dd.read_parquet(tmp, index='myindex')
-        assert_eq(ddf, ddf2)
+    ddf2 = dd.read_parquet(tmp, index='myindex')
+    assert_eq(ddf, ddf2)
 
 
-def test_read_parquet_custom_columns(engine):
-    with tmpdir() as tmp:
-        tmp = str(tmp)
-        data = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
-                             'f': np.arange(1000, dtype=np.float64)})
-        df = dd.from_pandas(data, chunksize=50)
-        df.to_parquet(tmp)
+def test_read_parquet_custom_columns(tmpdir, engine):
+    tmp = str(tmpdir)
+    data = pd.DataFrame({'i32': np.arange(1000, dtype=np.int32),
+                         'f': np.arange(1000, dtype=np.float64)})
+    df = dd.from_pandas(data, chunksize=50)
+    df.to_parquet(tmp)
 
-        df2 = dd.read_parquet(tmp, columns=['i32', 'f'], engine=engine)
-        assert_eq(df2, df2, check_index=False)
+    df2 = dd.read_parquet(tmp, columns=['i32', 'f'], engine=engine)
+    assert_eq(df2, df2, check_index=False)
 
-        df3 = dd.read_parquet(tmp, columns=['f', 'i32'], engine=engine)
-        assert_eq(df3, df3, check_index=False)
+    df3 = dd.read_parquet(tmp, columns=['f', 'i32'], engine=engine)
+    assert_eq(df3, df3, check_index=False)
 
 
 @pytest.mark.parametrize('df,write_kwargs,read_kwargs', [
@@ -358,19 +411,21 @@ def test_read_parquet_custom_columns(engine):
     (pd.DataFrame({'.': [3., 2., None]}), {}, {}),
     (pd.DataFrame({' ': [3., 2., None]}), {}, {}),
 ])
-def test_roundtrip(df, write_kwargs, read_kwargs):
-    with tmpdir() as tmp:
-        tmp = str(tmp)
-        if df.index.name is None:
-            df.index.name = 'index'
-        ddf = dd.from_pandas(df, npartitions=2)
+def test_roundtrip(tmpdir, df, write_kwargs, read_kwargs):
+    check_fastparquet()
+    tmp = str(tmpdir)
+    if df.index.name is None:
+        df.index.name = 'index'
+    ddf = dd.from_pandas(df, npartitions=2)
 
-        dd.to_parquet(ddf, tmp, **write_kwargs)
-        ddf2 = dd.read_parquet(tmp, index=df.index.name, **read_kwargs)
-        assert_eq(ddf, ddf2)
+    dd.to_parquet(ddf, tmp, **write_kwargs)
+    ddf2 = dd.read_parquet(tmp, index=df.index.name, **read_kwargs)
+    assert_eq(ddf, ddf2)
 
 
-def test_categories(fn):
+def test_categories(tmpdir):
+    check_fastparquet()
+    fn = str(tmpdir)
     df = pd.DataFrame({'x': [1, 2, 3, 4, 5],
                        'y': list('caaab')})
     ddf = dd.from_pandas(df, npartitions=2)
@@ -392,43 +447,44 @@ def test_categories(fn):
         ddf2 = dd.read_parquet(fn, categories=['foo'])
 
 
-def test_empty_partition(fn):
+def test_empty_partition(tmpdir, engine):
+    fn = str(tmpdir)
     df = pd.DataFrame({"a": range(10), "b": range(10)})
     ddf = dd.from_pandas(df, npartitions=5)
 
-    # fails as there are empty partitions
     ddf2 = ddf[ddf.a <= 5]
-    ddf2.to_parquet(fn)
+    ddf2.to_parquet(fn, engine=engine)
 
-    ddf3 = dd.read_parquet(fn)
-    assert_eq(ddf2.compute(), ddf3.compute(), check_names=False,
-              check_index=False)
+    ddf3 = dd.read_parquet(fn, engine=engine)
+    sol = ddf2.compute()
+    assert_eq(sol, ddf3, check_names=False, check_index=False)
 
 
-def test_timestamp_index():
-    with tmpfile() as fn:
-        df = tm.makeTimeDataFrame()
-        df.index.name = 'foo'
-        ddf = dd.from_pandas(df, npartitions=5)
-        ddf.to_parquet(fn)
-        ddf2 = dd.read_parquet(fn)
-        assert_eq(ddf, ddf2)
+def test_timestamp_index(tmpdir, engine):
+    fn = str(tmpdir)
+    df = tm.makeTimeDataFrame()
+    df.index.name = 'foo'
+    ddf = dd.from_pandas(df, npartitions=5)
+    ddf.to_parquet(fn, engine=engine)
+    ddf2 = dd.read_parquet(fn, engine=engine)
+    assert_eq(df, ddf2)
 
 
-@pytest.mark.skipif(not pyarrow, reason='pyarrow not found')
-def test_to_parquet_default_writes_nulls():
-    import pyarrow.parquet as pq
+def test_to_parquet_default_writes_nulls(tmpdir):
+    check_fastparquet()
+    check_pyarrow()
+    fn = str(tmpdir.join('test.parquet'))
 
     df = pd.DataFrame({'c1': [1., np.nan, 2, np.nan, 3]})
     ddf = dd.from_pandas(df, npartitions=1)
 
-    with tmpfile() as fn:
-        ddf.to_parquet(fn)
-        table = pq.read_table(fn)
-        assert table[1].null_count == 2
+    ddf.to_parquet(fn)
+    table = pq.read_table(fn)
+    assert table[1].null_count == 2
 
 
 def test_partition_on(tmpdir):
+    check_fastparquet()
     tmpdir = str(tmpdir)
     df = pd.DataFrame({'a': np.random.choice(['A', 'B', 'C'], size=100),
                        'b': np.random.random(size=100),
@@ -440,7 +496,9 @@ def test_partition_on(tmpdir):
         assert set(df.b[df.a == val]) == set(out.b[out.a == val])
 
 
-def test_filters(fn):
+def test_filters(tmpdir):
+    check_fastparquet()
+    fn = str(tmpdir)
 
     df = pd.DataFrame({'at': ['ab', 'aa', 'ba', 'da', 'bb']})
     ddf = dd.from_pandas(df, npartitions=1)
@@ -467,15 +525,9 @@ def test_filters(fn):
     assert len(ddf2) > 0
 
 
-def test_no_index(fn):
-    df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
-    ddf = dd.from_pandas(df, npartitions=2)
-    ddf.to_parquet(fn, write_index=False)
-    dd.read_parquet(fn).compute()
-
-
 @pytest.mark.parametrize('get', [dask.threaded.get, dask.multiprocessing.get])
 def test_to_parquet_lazy(tmpdir, get):
+    check_fastparquet()
     tmpdir = str(tmpdir)
     df = pd.DataFrame({'a': [1, 2, 3, 4],
                        'b': [1., 2., 3., 4.]})
@@ -484,7 +536,6 @@ def test_to_parquet_lazy(tmpdir, get):
     value = ddf.to_parquet(tmpdir, compute=False)
 
     assert hasattr(value, 'dask')
-    # assert not os.path.exists(tmpdir)
     value.compute(get=get)
     assert os.path.exists(tmpdir)
 
@@ -493,18 +544,9 @@ def test_to_parquet_lazy(tmpdir, get):
     assert_eq(ddf, ddf2)
 
 
-def test_empty(fn):
-    df = pd.DataFrame({'a': ['a', 'b', 'b'], 'b': [4, 5, 6]})[0:0]
-    ddf = dd.from_pandas(df, npartitions=2)
-    ddf.to_parquet(fn, write_index=False)
-    files = os.listdir(fn)
-    assert '_metadata' in files
-    out = dd.read_parquet(fn).compute()
-    assert len(out) == 0
-    assert_eq(out, df)
-
-
-def test_timestamp96(fn):
+def test_timestamp96(tmpdir):
+    check_fastparquet()
+    fn = str(tmpdir)
     df = pd.DataFrame({'a': ['now']}, dtype='M8[ns]')
     ddf = dd.from_pandas(df, 1)
     ddf.to_parquet(fn, write_index=False, times='int96')
@@ -514,7 +556,9 @@ def test_timestamp96(fn):
     assert_eq(out, df)
 
 
-def test_drill_scheme(fn):
+def test_drill_scheme(tmpdir):
+    check_fastparquet()
+    fn = str(tmpdir)
     N = 5
     df1 = pd.DataFrame({c: np.random.random(N)
                         for i, c in enumerate(['a', 'b', 'c'])})
@@ -537,7 +581,9 @@ def test_drill_scheme(fn):
     assert (np.unique(out.dir0) == ['test_data1', 'test_data2']).all()
 
 
-def test_parquet_select_cats(fn):
+def test_parquet_select_cats(tmpdir):
+    check_fastparquet()
+    fn = str(tmpdir)
     df = pd.DataFrame({
         'categories': pd.Series(
             np.random.choice(['a', 'b', 'c', 'd', 'e', 'f'], size=100),