Repository: dask/dask

Issue:
compression not passed to `engine.write_partition`
I'm trying to debug a separate issue (my files never start writing) and so I've been carefully reading parquet/core.py

If I'm understanding it correctly, the compression keyword is not passed to `engine.write_partition`


https://github.com/dask/dask/blob/master/dask/dataframe/io/parquet/core.py#L458-L481

```
    # write parts
    dwrite = delayed(engine.write_partition)
    parts = [
        dwrite(
            d,
            path,
            fs,
            filename,
            partition_on,
            write_metadata_file,
            fmd=meta,
            index_cols=index_cols,
            **kwargs_pass
        )
        for d, filename in zip(df.to_delayed(), filenames)
    ]

    # single task to complete
    out = delayed(lambda x: None)(parts)
    if write_metadata_file:
        out = delayed(engine.write_metadata)(
            parts, meta, fs, path, append=append, compression=compression
        )
```

`kwargs_pass` will not have compression in as it's a `_to_parquet_kwargs`

cc @rjzamora 


Discussion:
Looks to me like you are correct @birdsarah - This seems like a bug.  I will try to explore this soonish.

Resolving patch:
diff --git a/dask/dataframe/io/parquet/core.py b/dask/dataframe/io/parquet/core.py
--- a/dask/dataframe/io/parquet/core.py
+++ b/dask/dataframe/io/parquet/core.py
@@ -374,6 +374,12 @@ def to_parquet(
     """
     from dask import delayed
 
+    if compression == "default":
+        if snappy is not None:
+            compression = "snappy"
+        else:
+            compression = None
+
     partition_on = partition_on or []
     if isinstance(partition_on, string_types):
         partition_on = [partition_on]
@@ -385,11 +391,6 @@ def to_parquet(
             "columns=%s" % (str(partition_on), str(list(df.columns)))
         )
 
-    if compression != "default":
-        kwargs["compression"] = compression
-    elif snappy is not None:
-        kwargs["compression"] = "snappy"
-
     if isinstance(engine, str):
         engine = get_engine(engine)
 
@@ -466,6 +467,7 @@ def to_parquet(
             partition_on,
             write_metadata_file,
             fmd=meta,
+            compression=compression,
             index_cols=index_cols,
             **kwargs_pass
         )


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -1325,6 +1325,35 @@ def test_columns_name(tmpdir, engine):
     assert_eq(result, df)
 
 
+def check_compression(engine, filename, compression):
+    if engine == "fastparquet":
+        pf = fastparquet.ParquetFile(filename)
+        md = pf.fmd.row_groups[0].columns[0].meta_data
+        if compression is None:
+            assert md.total_compressed_size == md.total_uncompressed_size
+        else:
+            assert md.total_compressed_size != md.total_uncompressed_size
+    else:
+        metadata = pa.parquet.ParquetDataset(filename).metadata
+        names = metadata.schema.names
+        for i in range(metadata.num_row_groups):
+            row_group = metadata.row_group(i)
+            for j in range(len(names)):
+                column = row_group.column(j)
+                if compression is None:
+                    assert (
+                        column.total_compressed_size == column.total_uncompressed_size
+                    )
+                else:
+                    compress_expect = compression
+                    if compression == "default":
+                        compress_expect = "snappy"
+                    assert compress_expect.lower() == column.compression.lower()
+                    assert (
+                        column.total_compressed_size != column.total_uncompressed_size
+                    )
+
+
 @pytest.mark.parametrize("compression,", ["default", None, "gzip", "snappy"])
 def test_writing_parquet_with_compression(tmpdir, compression, engine):
     fn = str(tmpdir)
@@ -1338,6 +1367,7 @@ def test_writing_parquet_with_compression(tmpdir, compression, engine):
     ddf.to_parquet(fn, compression=compression, engine=engine)
     out = dd.read_parquet(fn, engine=engine)
     assert_eq(out, ddf)
+    check_compression(engine, fn, compression)
 
 
 @pytest.fixture(