Repository: dask/dask

Issue:
Appending to parquet file seems to break future appending
Dask 0.16.1, Fastparquet 0.1.3

This seems to be a dask problem as replicating the issue in fastparquet doesn't cause any kind of exception.

Appending to a hive parquet folder seems to break something and future appends fail. Not sure what's special about this use case (perhaps the datetimeindex?). Replication code and the exception are blow.

```python
from datetime import datetime

import dask.dataframe as dd
import fastparquet
import numpy as np
import pandas as pd


d_range = pd.date_range(start='2017-01-01', end='2017-01-02', freq='1h')
df = pd.DataFrame(np.random.randn(len(d_range), 1), index=d_range, columns=list('A'))
ddf = dd.from_pandas(df, npartitions=1)
fastparquet.write('test_parq_fp', df, file_scheme='hive')
ddf.to_parquet('test_parq_dask')

d_range = pd.date_range(start='2017-01-02', end='2017-01-03', freq='1h')
df = pd.DataFrame(np.random.randn(len(d_range), 1), index=d_range, columns=list('A'))
ddf = dd.from_pandas(df, npartitions=1)
fastparquet.write('test_parq_fp', df, append=True, file_scheme='hive')
ddf.to_parquet('test_parq_dask', append=True)

d_range = pd.date_range(start='2017-01-04', end='2017-01-05', freq='1h')
df = pd.DataFrame(np.random.randn(len(d_range), 1), index=d_range, columns=list('A'))
ddf = dd.from_pandas(df, npartitions=1)
fastparquet.write('test_parq_fp', df, append=True, file_scheme='hive')
ddf.to_parquet('test_parq_dask', append=True)
```

```
Traceback (most recent call last):
  File "break_parquet.py", line 26, in <module>
    ddf.to_parquet('test_parq_dask', append=True)
  File "/Users/shughes/miniconda3/lib/python3.6/site-packages/dask/dataframe/core.py", line 1010, in to_parquet
    return to_parquet(self, path, *args, **kwargs)
  File "/Users/shughes/miniconda3/lib/python3.6/site-packages/dask/dataframe/io/parquet.py", line 840, in to_parquet
    storage_options=storage_options, **kwargs)
  File "/Users/shughes/miniconda3/lib/python3.6/site-packages/dask/dataframe/io/parquet.py", line 420, in _write_fastparquet
    old_end = minmax[index_cols[0]]['max'][-1]
KeyError: 'index'
```


Discussion:
Can you try with https://github.com/dask/dask/pull/3097 ?
The following change in `fastparquet.api` seems to fix this.

```
L673                        all(mx <= mn for mx, mn in zip(max[:-1], min[1:]))):
```
(condition goes to "less than or equal")

I'd appreciate someone else's thoughts on whether this is the right fix.
```
In [8]: pd.date_range(start='2017-01-01', end='2017-01-02', freq='1h').max()
Out[8]: Timestamp('2017-01-02 00:00:00', freq='H')

In [9]: pd.date_range(start='2017-01-02', end='2017-01-03', freq='1h').min()
Out[9]: Timestamp('2017-01-02 00:00:00', freq='H')
```
... because times repeat, last entry in one partition versus first in next.

Note, you can always use `ignore_divisions=True`.
Tried  #3097 with no joy .  I guess my problem is that instead of throwing the usual 'Divisions overlap' exception it throws something much harder to understand.
Right, the case is not that the new data's divisions overlap with existing data, but that the original data doesn't have non-overlapping divisions in the first case. Except that they don't *quite* overlap, which is why I'm on the fence about whether this is a bug or not.
@martindurant is there something to fix here?  If so can you summarize it?  If not can you close?

Resolving patch:
diff --git a/dask/dataframe/io/parquet/arrow.py b/dask/dataframe/io/parquet/arrow.py
--- a/dask/dataframe/io/parquet/arrow.py
+++ b/dask/dataframe/io/parquet/arrow.py
@@ -633,7 +633,7 @@ def initialize_write(
                             break
 
                 divisions = division_info["divisions"]
-                if divisions[0] < old_end:
+                if divisions[0] <= old_end:
                     raise ValueError(
                         "Appended divisions overlapping with the previous ones"
                         " (set ignore_divisions=True to append anyway).\n"
diff --git a/dask/dataframe/io/parquet/fastparquet.py b/dask/dataframe/io/parquet/fastparquet.py
--- a/dask/dataframe/io/parquet/fastparquet.py
+++ b/dask/dataframe/io/parquet/fastparquet.py
@@ -1171,9 +1171,15 @@ def initialize_write(
                     ignore_divisions = True
             if not ignore_divisions:
                 minmax = fastparquet.api.sorted_partitioned_columns(pf)
-                old_end = minmax[index_cols[0]]["max"][-1]
+                # If fastparquet detects that a partitioned column isn't sorted, it won't
+                # appear in the resulting min/max dictionary
+                old_end = (
+                    minmax[index_cols[0]]["max"][-1]
+                    if index_cols[0] in minmax
+                    else None
+                )
                 divisions = division_info["divisions"]
-                if divisions[0] < old_end:
+                if old_end is None or divisions[0] <= old_end:
                     raise ValueError(
                         "Appended divisions overlapping with previous ones."
                         "\n"


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -842,22 +842,35 @@ def test_append_wo_index(tmpdir, engine, metadata_file):
 
 
 @pytest.mark.parametrize("metadata_file", [False, True])
-def test_append_overlapping_divisions(tmpdir, engine, metadata_file):
+@pytest.mark.parametrize(
+    ("index", "offset"),
+    [
+        (
+            pd.date_range("2022-01-01", "2022-01-02", periods=500)
+            .to_series()
+            .astype("datetime64[ms]"),
+            pd.Timedelta(days=1),
+        ),
+        (pd.RangeIndex(0, 500, 1), 499),
+    ],
+)
+def test_append_overlapping_divisions(tmpdir, engine, metadata_file, index, offset):
     """Test raising of error when divisions overlapping."""
     tmp = str(tmpdir)
+
     df = pd.DataFrame(
         {
-            "i32": np.arange(1000, dtype=np.int32),
-            "i64": np.arange(1000, dtype=np.int64),
-            "f": np.arange(1000, dtype=np.float64),
-            "bhello": np.random.choice(["hello", "yo", "people"], size=1000).astype(
-                "O"
-            ),
-        }
+            "i32": np.arange(len(index), dtype=np.int32),
+            "i64": np.arange(len(index), dtype=np.int64),
+            "f": np.arange(len(index), dtype=np.float64),
+            "bhello": np.random.choice(
+                ["hello", "yo", "people"], size=len(index)
+            ).astype("O"),
+        },
+        index=index,
     )
-    half = len(df) // 2
-    ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100)
-    ddf2 = dd.from_pandas(df.iloc[half - 10 :], chunksize=100)
+    ddf1 = dd.from_pandas(df, chunksize=100)
+    ddf2 = dd.from_pandas(df.set_index(df.index + offset), chunksize=100)
     ddf1.to_parquet(tmp, engine=engine, write_metadata_file=metadata_file)
 
     with pytest.raises(ValueError) as excinfo: