Repository: dask/dask

Issue:
to_parquet with pyarrow has produced duplicated files, after a worker has crashed
**What happened**:
Our job was writing 100 partitions, using the pyarrow engine. One of the workers crashed, after it has written a parquet file. As a consequence, we had 101 parquet files with duplicate data in our s3 bucket.

We observed, that the `to_parquet` task got re-scheduled and re-executed on an other worker, as it was in `memory` and waiting to be processed by the `write_metadata` task.

**What you expected to happen**:
100 parquet partitions in our s3 bucket, even when a worker dies after writing.

**Minimal Complete Verifiable Example**:
```python
import pandas as pd
from dask import dataframe

df = pd.DataFrame(["A", "B"], columns=["part"])
df['data'] = 1
ddf = dataframe.from_pandas(df, npartitions=1)

for i in range(3):
    ddf.to_parquet("out/fastparquet", engine="fastparquet", partition_on="part")
    ddf.to_parquet("out/pyarrow", engine="pyarrow", partition_on="part")
```

produces the following output

```
out
├── fastparquet
│   ├── _common_metadata
│   ├── _metadata
│   ├── part=A
│   │   └── part.0.parquet
│   └── part=B
│       └── part.0.parquet
└── pyarrow
    ├── _common_metadata
    ├── _metadata
    ├── part=A
    │   ├── 33a45f31937e467880cd24d11c055133.parquet
    │   ├── 691d5e18d5b745fe8cbb203cbf4f7523.parquet
    │   └── aa4a303bbaec44c1ae15c92cbcb1dfad.parquet
    └── part=B
        ├── 6794a35ed580479d90087c4b13d41442.parquet
        ├── d0847e218a9142429014300bfb7f1ee9.parquet
        └── f526b99981a34595a9bbec220e3d21e4.parquet
```



Discussion:

Resolving patch:
diff --git a/dask/dataframe/io/parquet/arrow.py b/dask/dataframe/io/parquet/arrow.py
--- a/dask/dataframe/io/parquet/arrow.py
+++ b/dask/dataframe/io/parquet/arrow.py
@@ -7,9 +7,7 @@
 import pandas as pd
 import pyarrow as pa
 import pyarrow.parquet as pq
-from pyarrow.compat import guid
 from ....utils import getargspec
-
 from ..utils import _get_pyarrow_dtypes, _meta_from_dtypes
 from ...utils import clear_known_categories
 from ....core import flatten
@@ -29,7 +27,9 @@
 #
 
 
-def _write_partitioned(table, root_path, partition_cols, fs, index_cols=(), **kwargs):
+def _write_partitioned(
+    table, root_path, filename, partition_cols, fs, index_cols=(), **kwargs
+):
     """ Write table to a partitioned dataset with pyarrow.
 
         Logic copied from pyarrow.parquet.
@@ -73,11 +73,10 @@ def _write_partitioned(table, root_path, partition_cols, fs, index_cols=(), **kw
         )
         prefix = fs.sep.join([root_path, subdir])
         fs.mkdir(prefix, exists_ok=True)
-        outfile = guid() + ".parquet"
-        full_path = fs.sep.join([prefix, outfile])
+        full_path = fs.sep.join([prefix, filename])
         with fs.open(full_path, "wb") as f:
             pq.write_table(subtable, f, metadata_collector=md_list, **kwargs)
-        md_list[-1].set_file_path(fs.sep.join([subdir, outfile]))
+        md_list[-1].set_file_path(fs.sep.join([subdir, filename]))
 
     return md_list
 
@@ -933,7 +932,7 @@ def write_partition(
         t = pa.Table.from_pandas(df, preserve_index=preserve_index, schema=schema)
         if partition_on:
             md_list = _write_partitioned(
-                t, path, partition_on, fs, index_cols=index_cols, **kwargs
+                t, path, filename, partition_on, fs, index_cols=index_cols, **kwargs
             )
             if md_list:
                 _meta = md_list[0]


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -1096,6 +1096,34 @@ def test_partition_on(tmpdir, engine):
         assert set(df.b[df.a2 == val]) == set(out.b[out.a2 == val])
 
 
+def test_partition_on_duplicates(tmpdir, engine):
+    # https://github.com/dask/dask/issues/6445
+    tmpdir = str(tmpdir)
+    df = pd.DataFrame(
+        {
+            "a1": np.random.choice(["A", "B", "C"], size=100),
+            "a2": np.random.choice(["X", "Y", "Z"], size=100),
+            "data": np.random.random(size=100),
+        }
+    )
+    d = dd.from_pandas(df, npartitions=2)
+
+    for _ in range(2):
+        d.to_parquet(tmpdir, partition_on=["a1", "a2"], engine=engine)
+
+    out = dd.read_parquet(tmpdir, engine=engine).compute()
+
+    assert len(df) == len(out)
+    for root, dirs, files in os.walk(tmpdir):
+        for file in files:
+            assert file in (
+                "part.0.parquet",
+                "part.1.parquet",
+                "_common_metadata",
+                "_metadata",
+            )
+
+
 @pytest.mark.parametrize("partition_on", ["aa", ["aa"]])
 def test_partition_on_string(tmpdir, partition_on):
     tmpdir = str(tmpdir)