Repository: dask/dask

Issue:
to_parquet failing on pyarrow master when different partitions result in different parquet schema
In pyarrow master, we made a change to `append_row_groups` to require equal schema's when appending (https://issues.apache.org/jira/browse/ARROW-8062, 
https://github.com/apache/arrow/pull/7180). But that apparently causes test failures in the dask test suite:

<details>

```
2020-05-26T06:57:29.5507643Z =================================== FAILURES ===================================
2020-05-26T06:57:29.5509731Z _ test_to_parquet_pyarrow_w_inconsistent_schema_by_partition_fails_by_default __
2020-05-26T06:57:29.5509974Z 
2020-05-26T06:57:29.5512116Z tmpdir = local('/tmp/pytest-of-root/pytest-0/test_to_parquet_pyarrow_w_inco0')
2020-05-26T06:57:29.5512377Z 
2020-05-26T06:57:29.5514934Z     def test_to_parquet_pyarrow_w_inconsistent_schema_by_partition_fails_by_default(tmpdir):
2020-05-26T06:57:29.5515252Z         check_pyarrow()
2020-05-26T06:57:29.5515584Z     
2020-05-26T06:57:29.5515752Z         df = pd.DataFrame(
2020-05-26T06:57:29.5515998Z             {"partition_column": [0, 0, 1, 1], "strings": ["a", "b", None, None]}
2020-05-26T06:57:29.5516182Z         )
2020-05-26T06:57:29.5516369Z     
2020-05-26T06:57:29.5516616Z         ddf = dd.from_pandas(df, npartitions=2)
2020-05-26T06:57:29.5516806Z >       ddf.to_parquet(str(tmpdir), engine="pyarrow", partition_on=["partition_column"])
2020-05-26T06:57:29.5516894Z 
2020-05-26T06:57:29.5519255Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/io/tests/test_parquet.py:965: 
2020-05-26T06:57:29.5519532Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2020-05-26T06:57:29.5520327Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/core.py:3811: in to_parquet
2020-05-26T06:57:29.5525402Z     return to_parquet(self, path, *args, **kwargs)
2020-05-26T06:57:29.5528566Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/io/parquet/core.py:461: in to_parquet
2020-05-26T06:57:29.5529800Z     out = out.compute(**compute_kwargs)
2020-05-26T06:57:29.5530394Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/base.py:166: in compute
2020-05-26T06:57:29.5530739Z     (result,) = compute(self, traverse=False, **kwargs)
2020-05-26T06:57:29.5531158Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/base.py:444: in compute
2020-05-26T06:57:29.5539865Z     results = schedule(dsk, keys, **kwargs)
2020-05-26T06:57:29.5543266Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/threaded.py:84: in get
2020-05-26T06:57:29.5545361Z     **kwargs
2020-05-26T06:57:29.5547293Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/local.py:486: in get_async
2020-05-26T06:57:29.5549251Z     raise_exception(exc, tb)
2020-05-26T06:57:29.5550734Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/local.py:316: in reraise
2020-05-26T06:57:29.5552431Z     raise exc
2020-05-26T06:57:29.5553073Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/local.py:222: in execute_task
2020-05-26T06:57:29.5553366Z     result = _execute_task(task, data)
2020-05-26T06:57:29.5553764Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/core.py:121: in _execute_task
2020-05-26T06:57:29.5553938Z     return func(*(_execute_task(a, cache) for a in args))
2020-05-26T06:57:29.5554268Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/utils.py:30: in apply
2020-05-26T06:57:29.5554462Z     return func(*args, **kwargs)
2020-05-26T06:57:29.5554815Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/io/parquet/arrow.py:763: in write_metadata
2020-05-26T06:57:29.5554989Z     _meta.append_row_groups(parts[i][0]["meta"])
2020-05-26T06:57:29.5555156Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2020-05-26T06:57:29.5555299Z 
2020-05-26T06:57:29.5555451Z >   ???
2020-05-26T06:57:29.5555610Z E   RuntimeError: AppendRowGroups requires equal schemas.
2020-05-26T06:57:29.5555722Z 
2020-05-26T06:57:29.5555859Z pyarrow/_parquet.pyx:687: RuntimeError
2020-05-26T06:57:29.5556038Z ___________________ test_timeseries_nulls_in_schema[pyarrow] ___________________
2020-05-26T06:57:29.5556155Z 
2020-05-26T06:57:29.5556469Z tmpdir = local('/tmp/pytest-of-root/pytest-0/test_timeseries_nulls_in_schem0')
2020-05-26T06:57:29.5556765Z engine = 'pyarrow'
2020-05-26T06:57:29.5556881Z 
2020-05-26T06:57:29.5557045Z     def test_timeseries_nulls_in_schema(tmpdir, engine):
2020-05-26T06:57:29.5557218Z         # GH#5608: relative path failing _metadata/_common_metadata detection.
2020-05-26T06:57:29.5557373Z         tmp_path = str(tmpdir.mkdir("files"))
2020-05-26T06:57:29.5557554Z         tmp_path = os.path.join(tmp_path, "../", "files")
2020-05-26T06:57:29.5557698Z     
2020-05-26T06:57:29.5557871Z         ddf2 = (
2020-05-26T06:57:29.5558194Z             dask.datasets.timeseries(start="2000-01-01", end="2000-01-03", freq="1h")
2020-05-26T06:57:29.5558363Z             .reset_index()
2020-05-26T06:57:29.5558526Z             .map_partitions(lambda x: x.loc[:5])
2020-05-26T06:57:29.5558701Z         )
2020-05-26T06:57:29.5558847Z         ddf2 = ddf2.set_index("x").reset_index().persist()
2020-05-26T06:57:29.5559172Z         ddf2.name = ddf2.name.where(ddf2.timestamp == "2000-01-01", None)
2020-05-26T06:57:29.5559356Z     
2020-05-26T06:57:29.5559509Z >       ddf2.to_parquet(tmp_path, engine=engine)
2020-05-26T06:57:29.5559623Z 
2020-05-26T06:57:29.5560119Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/io/tests/test_parquet.py:2011: 
2020-05-26T06:57:29.5560347Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2020-05-26T06:57:29.5560754Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/core.py:3811: in to_parquet
2020-05-26T06:57:29.5561517Z     return to_parquet(self, path, *args, **kwargs)
2020-05-26T06:57:29.5562388Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/io/parquet/core.py:461: in to_parquet
2020-05-26T06:57:29.5562518Z     out = out.compute(**compute_kwargs)
2020-05-26T06:57:29.5562799Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/base.py:166: in compute
2020-05-26T06:57:29.5562906Z     (result,) = compute(self, traverse=False, **kwargs)
2020-05-26T06:57:29.5563189Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/base.py:444: in compute
2020-05-26T06:57:29.5563309Z     results = schedule(dsk, keys, **kwargs)
2020-05-26T06:57:29.5563699Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/threaded.py:84: in get
2020-05-26T06:57:29.5563815Z     **kwargs
2020-05-26T06:57:29.5564074Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/local.py:486: in get_async
2020-05-26T06:57:29.5564193Z     raise_exception(exc, tb)
2020-05-26T06:57:29.5564468Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/local.py:316: in reraise
2020-05-26T06:57:29.5564589Z     raise exc
2020-05-26T06:57:29.5564881Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/local.py:222: in execute_task
2020-05-26T06:57:29.5565003Z     result = _execute_task(task, data)
2020-05-26T06:57:29.5565286Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/core.py:121: in _execute_task
2020-05-26T06:57:29.5565410Z     return func(*(_execute_task(a, cache) for a in args))
2020-05-26T06:57:29.5565688Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/utils.py:30: in apply
2020-05-26T06:57:29.5565789Z     return func(*args, **kwargs)
2020-05-26T06:57:29.5566173Z opt/conda/envs/arrow/lib/python3.7/site-packages/dask/dataframe/io/parquet/arrow.py:763: in write_metadata
2020-05-26T06:57:29.5566299Z     _meta.append_row_groups(parts[i][0]["meta"])
2020-05-26T06:57:29.5566410Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2020-05-26T06:57:29.5566474Z 
2020-05-26T06:57:29.5566570Z >   ???
2020-05-26T06:57:29.5566675Z E   RuntimeError: AppendRowGroups requires equal schemas.
2020-05-26T06:57:29.5566733Z 
2020-05-26T06:57:29.5566831Z pyarrow/_parquet.pyx:687: RuntimeError
```

</details>


I when doing `FileMetadata.append_row_groups`, it *seems* to make sense that we require the schemas to be the same, because afterwards there is only a single schema.

Looking at the failures in dask, it are cases where the different partitions of the dask dataframe result in a different pyarrow/parquet schema (and specifically a partition with all nulls in an object dtype (string) column). 

Do you see a solution for this on the dask side? (in principle there is the `meta` dtypes, but here of course it's the typical problem that "object" dtype is not specific enough ..).

Or should we revert back to being less strict on Arrow's side?

cc @rjzamora @martindurant 



Discussion:
How are the schemas different, are the equivalent from pandas' point of view? The file metadata should contain the original pandas schema, which may be more precise than the parquet one, but maybe doesn't map perfectly to arrow's types, especially for object/str.
For example the first failing test does this:

```
    df = pd.DataFrame(
        {"partition_column": [0, 0, 1, 1], "strings": ["a", "b", None, None]}
    )

    ddf = dd.from_pandas(df, npartitions=2)
    ddf.to_parquet(str(tmpdir), engine="pyarrow", partition_on=["partition_column"])
```

So the first partition converts to pyarrow with a `string`-type column, the second with a `null`-type columns (because pyarrow infers a all-None object array as `null` type). 
So `to_parquet` is writing parquet files with two different schema's, and when combining those into a single `_metadata` object, we ignored this mismatch before, but now started to raise on this (`FileMetadata.append_row_groups(other_file_metadata)` now checks the schema to be equal).
So in practice this will only happen with object dtype columns (I think), but given that strings are stored in object dtype, quite an important use case ...
Hopefully there is no failure if the desired schema is passed as a kwarg directly to the `to_parquet` call in dask? (If it is not easy for you to try, I will give it a go as soon as I am set up)
So in the pandas schema, both parts just get "object" anyway... When do we get nullable string columns in pandas by default? :)
> Hopefully there is no failure if the desired schema is passed as a kwarg directly to the to_parquet call in dask?

Yes that still works. But it is a bit awkward to need to pass this ...

Now, I suppose this is not really solvable from dask's side when you have such all-missing object-dtype partitions. Unless you would assume that object dtype always means string, but that will for sure also break people's code.

> When do we get nullable string columns in pandas by default? :)

When more people start contributing to that part of pandas ;) (otherwise it will still take a while)
Regarding the nullable string dtype -> that's acually an easier workaround for users to do than specifying the full schema (they can convert the object column to string dtype, and then all should work).
> the first partition converts to pyarrow with a string-type column, the second with a null-type columns (because pyarrow infers a all-None object array as null type).

Maybe if a schema isn't passed dask could infer a common schema from the first few partitions? Similar to the `csv.Sniffer` class? I think an inferred null type column would be a pretty good indicator of an incorrect type in ~all cases.
...or maybe there could be a post-processing step where all partitions are cast to the common schema before being concatenated.

Or `pyarrow` allow a `cast_null=True` argument to coerce any null-type columns.
Why isn't dask writing the file with the correct schema? Dask knows the correct dtypes for all columns, can that be specified when writing?
> Why isn't dask writing the file with the correct schema? Dask knows the correct dtypes for all columns, can that be specified when writing?

The pandas dtypes should all be the same.  The problem is that the arrow type for a pandas "object" column depends on the null content.  The user **can** specify the schema when calling `to_parquet`, but I agree we should probably be able to handle inconsistent schemas in a more automated way.
> the arrow type for a pandas "object" column depends on the null content.

It actually depends on the full content, not only of nulls, as you could in principle also have integers or other objects in an object dtype column. But the nulls issue with strings is probably the most common.

@dhirschfeld's suggestion to "infer" a schema from a small part of the dask.dataframe in case no schema is otherwise specified by the user, could be a possible solution.  
Dask could eg convert the first x rows to a pyarrow table to get the schema, and then specify that same schema for writing all partitions to parquet. 

Or more sophisticated, it could in principle only infer it for the object dtype columns, as all other columns should have a non-ambiguous mapping to arrow types (and for those you can create a schema from the empty `meta` dataframe).

Of course, if you would then have a string column that *starts* with a lot of nulls, you would still run into troubles. 
The ultra-cautious approach would be to get the arrow schema from every partition and find the common denominator before writing, but that could cause two passes through the whole of the input data.
Sorry to keep harping on this, but why do we need to infer anything? Why can't dask write with the correct schema?

```python
In [21]: schema = pa.Table.from_pandas(ddf._meta_nonempty).schema

In [22]: ddf.to_parquet(str(tmpdir), engine="pyarrow", partition_on=["partition_column"], schema=schema)

In [23]: dd.read_parquet(str(tmpdir), engine="pyarrow")
Out[23]:
Dask DataFrame Structure:
              strings partition_column
npartitions=6
               object  category[known]
                  ...              ...
...               ...              ...
                  ...              ...
                  ...              ...
Dask Name: read-parquet, 6 tasks
```

In this case, the only "inference" is the use of `_meta_nonempty` assuming that `object` dtype means strings, which is probably does.
Ah, I assumed `meta_nonempty` is not always available (and mentioned above that `meta` itself is not enough for object dtype). But yes, if that is always available, then that seems fine to use. In the end we just need a schema, wether it is from the `meta_nonempty`, or from the first few rows). 

I assume that the code that creates `meta_nonempty` simply assumes that `object` dtype means strings? 
> In this case, the only "inference" is the use of _meta_nonempty assuming that object dtype means strings, which is probably does.

"Probably" is a problem, though! 
If _meta_nonempty is the head of the dataframe, it might well be null, for instance. We certainly don't want to assume str for every object.
Note that fastparquet has an object_encoding kwarg which can be set per column, like a light version of making a whole schema; it allows, for instance, for JSON-encoding python dicts.
> We certainly don't want to assume str for every object.

That seems to be the case, though, for meta_nonempty:

https://github.com/dask/dask/blob/3996e914b7ef846273eda9ff4c2d4d850ae5018f/dask/dataframe/utils.py#L490-L499
Bahh, I didn't know that. There should surely be a warning around this and pointer to the user on how to make their own schema for the fail case.
I'd recommend that we write the data with `_meta_nonempty` when no schema is provided by the user. And document the edge cases around object-dtype if it isn't already.

But to answer @jorisvandenbossche's original question:
  
> Or should we revert back to being less strict on Arrow's side?

I think the strictness is a a good goal, but this will cause files written by older versions of Dask to not be readable by Dask + new pyarrows. Whether that's acceptable should be determined by pyarrow's backwards-compatibility policies.
> this will cause files written by older versions of Dask to not be readable by Dask + new pyarrows.

It's only about writing (it's only then that dask calls `append_row_group` to create the `_metadata` file), so it won't render existing files unreadable.

But existing code that writes dataframes with object dtypes with differently inferred partitions might start raising an error with future pyarrow.

> Whether that's acceptable should be determined by pyarrow's backwards-compatibility policies.

I think we would consider it a bug fix on pyarrow's side. So I think the only reason we would revert it is if the impact for dask is too big, so would appreciate some judgement from the dask maintainers on that ;)

I am not sure if we could eg only ignore a type mismatch if it involves a null type, and still raise for any other type mismatch (will check that). That would also solve the primary case in dask.


Special-casing null type seems like a reasonable thing to do, given that pandas can't actually contain that (is there a null extension type yet??).
I had some more discussion with Francois on the pyarrow side, and we would prefer to keep the current behaviour of pyarrow master. 
The problem is that otherwise we allow to create an invalid metadata file, where certain row groups actually have a different schema as the schema in that FileMetaData object. 

Now, to come back to one of the ideas that has been suggested earlier in this thread: would it be possible for dask to infer the schema from (a part of) one partition, and then use that schema for all partitions in the conversion to a pyarrow Table for writing to parquet (in case the user did not provide a schema manually) ? 
Apart from fixing the issue at hand, this also in general ensures that all written parquet files have a consistent schema.

The problem is then probably: how to determine / infer the schema?

- From the `meta.dtype` / `_meta_nonempty`: but this would basically mean defaulting to string for object dtype. 
- From (a part of) the first partition, so we can use actual data to infer (this is actually only needed for object dtype columns, as all other dtypes have a 1:1 mapping to array type). However, this might not be a good solution in general? (even getting the first few rows of an object dtype column might trigger a set of costly computations if the dataframe is a result of a long chain of tasks?)
- To help specifically with the object dtype column, dask could also add a `object_encoding` keyword that @martindurant mentioned (from fastparquet). It might something to consider for pyarrow as well, but in the meantime, I think dask could also add this (and this can prevent that the user needs to pass a full schema when the default inferred is not correct, but instead can only specify a type for the object columns)
Any follow-up here?
> would it be possible for dask to infer the schema from (a part of) one partition, and then use that schema for all partitions in the conversion to a pyarrow Table for writing to parquet (in case the user did not provide a schema manually) ?

To clarify, this is at write time?

I think we just use `DataFrame._meta_nonempty` to get the dtypes. That seems most consistent with the rest of Dask. If it's incorrect then the user has the option of providing the schema explicitly.
Yes, this is all about writing.

I am fine that, but I don't know how many people are using non-string object dtype columns with dask (as those will all start to fail by default, eg json like dicts, decimals, ..). @martindurant above said "We certainly don't want to assume str for every object.". 
If we go that way, it might still be good to have an easy way to specify which type to use for a specific column (instead of having to specify the full pyarrow schema for all columns).

Inferring from eg a `head(5)` might be problematic?



> Inferring from eg a head(5) might be problematic?

I'd prefer to avoid that.

> If we go that way, it might still be good to have an easy way to specify which type to use for a specific column

An `object_encoding` parameter that takes a dict of column name to dtype seems fine. Then we get the schema from `meta_nonempty` and update the specified columns with that user provided type.
> I think we just use DataFrame._meta_nonempty to get the dtypes. That seems most consistent with the rest of Dask. If it's incorrect then the user has the option of providing the schema explicitly.

I am also thinking we may need to "do our best" with `_meta_nonempty`, and require the user to be more explicit if it fails.  I'd rather avoid the need to parse any real file data (by default, at least). 
> An object_encoding parameter that takes a dict of column name to dtype seems fine.

You mean arrow or parquet type?
Not sure, what do you think?

On Tue, Jun 16, 2020 at 8:42 AM Martin Durant <notifications@github.com>
wrote:

> An object_encoding parameter that takes a dict of column name to dtype
> seems fine.
>
> You mean arrow or parquet type?
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/6243#issuecomment-644774169>, or
> unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AAKAOIUXFTLYIWFYXGV3L73RW5ZFPANCNFSM4NKLKOXQ>
> .
>

The fastparquet method was to pick explicitly from the parquet encodings; this is the most exact, but I'm not sure that arrow would make it easy. Mapping to arrow types is probably more obvious for the user that knows little about the vagaries of parquet; I don't know what you would loose. Since we are proposing a way around making whole arrow schemas, this latter way, `{col: arrow_type}` seems fairly straight-forward.

My point was, really, that "dtype" (in the pandas sense) isn't enough.
> > An object_encoding parameter that takes a dict of column name to dtype seems fine.
>
> You mean arrow or parquet type?

It would indeed need to be an arrow type, because 1) that's what is needed for the intermediate pandas->pyarrow.Table conversion and 2) pyarrow doesn't have a way to specify a certain parquet type when writing a pyarrow Table AFAIK.

---

Now, I am having a possible "insight" (not fully sure yet I am not missing something ;-)). So one of the tests that is failing, is actually testing that the written parquet dataset is *failing on read*. The reason this test is failing, is because it is now already *failing on write*. 
But so that in itself is not necessarily a bad change? 


Failing test:

<details>

https://github.com/dask/dask/blob/e7e3b735d4cfee04d44cf7054b122bec1569dd23/dask/dataframe/io/tests/test_parquet.py#L955-L974

</details>
> But so that in itself is not necessarily a bad change?

Good point - The test is expected to fail when reading back an inconsistent schema.  If pyarrow is now preventing the user from writing the inconsistent schema to begin with, there is really no loss in functionality here :)
Agree that that test should be failing on write, so long as we make sure the schemas are inconsistent (before implementing any suggestions here)
Now, I am certainly still missing something. As the second that test is failing, is actually testing that such an inconsistently written dataset *can be read*. But I don't yet fully understand what the difference is between both tests (why it asserts is fails in the one case and it works in the other case).

Second failing test:

<details>

https://github.com/dask/dask/blob/e7e3b735d4cfee04d44cf7054b122bec1569dd23/dask/dataframe/io/tests/test_parquet.py#L1998-L2022

</details>
So the first test is specifying a `dataset={"validate_schema": True},` in `read_parquet`, and the second test is not. So that clarifies the difference. 

Thus, forget my "insight" from above. Such datasets written with incosistent schemas in the different parquet files, right now read fine by default. This was actually fixed by @rjzamora in https://github.com/dask/dask/pull/5307, by ensuring to pass `validate_schema=False` when not specified by the user (with similar comments about the problem of object dtype partitions with all nulls .. https://github.com/dask/dask/pull/5307#issuecomment-525787069)
FWIW this is an issue when converting the NYCTaxi dataset to Parquet

```python
import dask.dataframe as dd
df = dd.read_csv(
    "s3://nyc-tlc/trip data/yellow_tripdata_2019-*.csv", 
    parse_dates=["tpep_pickup_datetime", "tpep_dropoff_datetime"],
    dtype={
        "VendorID": "UInt8",
        "passenger_count": "UInt8",
        "RatecodeID": "UInt8",
        "store_and_fwd_flag": "category",
        "PULocationID": "UInt16",
        "DOLocationID": "UInt16",
        "payment_type": "UInt8",
    },
)

df.to_parquet("foo.parquet")
```
FYI, this also happens when reading a dataframe with some empty partition. Something like this:

```
from dask import from_pandas
from pandas import DataFrame, to_datetime

df = DataFrame({
     "index": ["1", "2", "3", "2", "3", "1", "4"],
     "date":  to_datetime(["2017-01-01", "2017-01-01", "2017-01-01", "2017-01-02", "2017-01-02", 
                                            "2017-01-06", "2017-01-09"]),
     "amount": [100, 200, 300, 400, 500, 600, 700]
})
df = from_pandas(df, npartitions=2).set_index("index")
```
If `from_pandas` doesn't actually partition this into 2 partitions (BTW, why isn't this guaranteed?), which can be checked with `df.map_partitions(len).compute()`, the above error with `AppendRowGroups` is also raised.

With `npartitions=1` it goes fine of course, but maybe it's related. Hope this helps.


That is a much nicer reproducer :)  Thanks @jorloplaz 
> BTW, why isn't this guaranteed?

I believe the sequence of operations went like:
- Dask partitioned the original data equally, with two partitions of length 4 and 3
- To find the partitions after reindex, Dask did "approximate quantiles" on the index column (which is text type), and found (min, median, max) = ("1", "1", "4"). I'm not sure how these values arise
- Then the data was repartitioned, the first partition getting rows "1" <= index < "1" (i.e., none of them)



Resolving patch:
diff --git a/dask/dataframe/io/parquet/arrow.py b/dask/dataframe/io/parquet/arrow.py
--- a/dask/dataframe/io/parquet/arrow.py
+++ b/dask/dataframe/io/parquet/arrow.py
@@ -11,6 +11,7 @@
 from ..utils import _get_pyarrow_dtypes, _meta_from_dtypes
 from ...utils import clear_known_categories
 from ....core import flatten
+from dask import delayed
 
 from .utils import (
     _parse_pandas_metadata,
@@ -20,6 +21,7 @@
 )
 
 preserve_ind_supported = pa.__version__ >= LooseVersion("0.15.0")
+schema_field_supported = pa.__version__ >= LooseVersion("0.15.0")
 
 
 #
@@ -27,6 +29,20 @@
 #
 
 
+def _append_row_groups(metadata, md):
+    try:
+        metadata.append_row_groups(md)
+    except RuntimeError as err:
+        if "requires equal schemas" in str(err):
+            raise RuntimeError(
+                "Schemas are inconsistent, try using "
+                '`to_parquet(..., schema="infer")`, or pass an explicit '
+                "pyarrow schema."
+            )
+        else:
+            raise err
+
+
 def _write_partitioned(
     table, root_path, filename, partition_cols, fs, index_cols=(), **kwargs
 ):
@@ -239,7 +255,7 @@ def _gather_metadata(
             elif fn:
                 md.set_file_path(fn)
             if metadata:
-                metadata.append_row_groups(md)
+                _append_row_groups(metadata, md)
             else:
                 metadata = md
         return (
@@ -821,8 +837,51 @@ def initialize_write(
         partition_on=None,
         ignore_divisions=False,
         division_info=None,
+        schema=None,
+        index_cols=None,
         **kwargs,
     ):
+        # Infer schema if "infer"
+        # (also start with inferred schema if user passes a dict)
+        if schema == "infer" or isinstance(schema, dict):
+
+            # Start with schema from _meta_nonempty
+            _schema = pa.Schema.from_pandas(
+                df._meta_nonempty.set_index(index_cols)
+                if index_cols
+                else df._meta_nonempty
+            )
+
+            # Use dict to update our inferred schema
+            if isinstance(schema, dict):
+                schema = pa.schema(schema)
+                for name in schema.names:
+                    i = _schema.get_field_index(name)
+                    j = schema.get_field_index(name)
+                    _schema = _schema.set(i, schema.field(j))
+
+            # If we have object columns, we need to sample partitions
+            # until we find non-null data for each column in `sample`
+            sample = [col for col in df.columns if df[col].dtype == "object"]
+            if schema_field_supported and sample and schema == "infer":
+                delayed_schema_from_pandas = delayed(pa.Schema.from_pandas)
+                for i in range(df.npartitions):
+                    # Keep data on worker
+                    _s = delayed_schema_from_pandas(
+                        df[sample].to_delayed()[i]
+                    ).compute()
+                    for name, typ in zip(_s.names, _s.types):
+                        if typ != "null":
+                            i = _schema.get_field_index(name)
+                            j = _s.get_field_index(name)
+                            _schema = _schema.set(i, _s.field(j))
+                            sample.remove(name)
+                    if not sample:
+                        break
+
+            # Final (inferred) schema
+            schema = _schema
+
         dataset = fmd = None
         i_offset = 0
         if append and division_info is None:
@@ -906,7 +965,7 @@ def initialize_write(
                         "Previous: {} | New: {}".format(old_end, divisions[0])
                     )
 
-        return fmd, i_offset
+        return fmd, schema, i_offset
 
     @staticmethod
     def write_partition(
@@ -937,7 +996,7 @@ def write_partition(
             if md_list:
                 _meta = md_list[0]
                 for i in range(1, len(md_list)):
-                    _meta.append_row_groups(md_list[i])
+                    _append_row_groups(_meta, md_list[i])
         else:
             md_list = []
             with fs.open(fs.sep.join([path, filename]), "wb") as fil:
@@ -977,6 +1036,6 @@ def write_metadata(parts, fmd, fs, path, append=False, **kwargs):
                 _meta = parts[0][0]["meta"]
                 i_start = 1
             for i in range(i_start, len(parts)):
-                _meta.append_row_groups(parts[i][0]["meta"])
+                _append_row_groups(_meta, parts[i][0]["meta"])
             with fs.open(metadata_path, "wb") as fil:
                 _meta.write_metadata_file(fil)
diff --git a/dask/dataframe/io/parquet/core.py b/dask/dataframe/io/parquet/core.py
--- a/dask/dataframe/io/parquet/core.py
+++ b/dask/dataframe/io/parquet/core.py
@@ -98,7 +98,7 @@ def read_parquet(
     gather_statistics=None,
     split_row_groups=None,
     chunksize=None,
-    **kwargs
+    **kwargs,
 ):
     """
     Read a Parquet file into a Dask DataFrame
@@ -236,7 +236,7 @@ def read_parquet(
         gather_statistics=gather_statistics,
         filters=filters,
         split_row_groups=split_row_groups,
-        **kwargs
+        **kwargs,
     )
 
     # Parse dataset statistics from metadata (if available)
@@ -301,7 +301,8 @@ def to_parquet(
     write_metadata_file=True,
     compute=True,
     compute_kwargs=None,
-    **kwargs
+    schema=None,
+    **kwargs,
 ):
     """Store Dask.dataframe to Parquet files
 
@@ -345,6 +346,16 @@ def to_parquet(
         then a ``dask.delayed`` object is returned for future computation.
     compute_kwargs : dict, optional
         Options to be passed in to the compute method
+    schema : Schema object, dict, or {"infer", None}, optional
+        Global schema to use for the output dataset. Alternatively, a `dict`
+        of pyarrow types can be specified (e.g. `schema={"id": pa.string()}`).
+        For this case, fields excluded from the dictionary will be inferred
+        from `_meta_nonempty`.  If "infer", the first non-empty and non-null
+        partition will be used to infer the type for "object" columns. If
+        None (default), we let the backend infer the schema for each distinct
+        output partition. If the partitions produce inconsistent schemas,
+        pyarrow will throw an error when writing the shared _metadata file.
+        Note that this argument is ignored by the "fastparquet" engine.
     **kwargs :
         Extra options to be passed on to the specific backend.
 
@@ -404,6 +415,14 @@ def to_parquet(
                 "will be set to the index (and renamed to None)."
             )
 
+    # There are some "resrved" names that may be used as the default column
+    # name after resetting the index. However, we don't want to treat it as
+    # a "special" name if the string is already used as a "real" column name.
+    reserved_names = []
+    for name in ["index", "level_0"]:
+        if name not in df.columns:
+            reserved_names.append(name)
+
     # If write_index==True (default), reset the index and record the
     # name of the original index in `index_cols` (we will set the name
     # to the NONE_LABEL constant if it is originally `None`).
@@ -418,7 +437,9 @@ def to_parquet(
         none_index = list(df._meta.index.names) == [None]
         df = df.reset_index()
         if none_index:
-            df.columns = [c if c != "index" else NONE_LABEL for c in df.columns]
+            df.columns = [
+                c if c not in reserved_names else NONE_LABEL for c in df.columns
+            ]
         index_cols = [c for c in set(df.columns).difference(real_cols)]
     else:
         # Not writing index - might as well drop it
@@ -439,7 +460,7 @@ def to_parquet(
 
     # Engine-specific initialization steps to write the dataset.
     # Possibly create parquet metadata, and load existing stuff if appending
-    meta, i_offset = engine.initialize_write(
+    meta, schema, i_offset = engine.initialize_write(
         df,
         fs,
         path,
@@ -448,7 +469,8 @@ def to_parquet(
         partition_on=partition_on,
         division_info=division_info,
         index_cols=index_cols,
-        **kwargs_pass
+        schema=schema,
+        **kwargs_pass,
     )
 
     # Use i_offset and df.npartitions to define file-name list
@@ -467,7 +489,8 @@ def to_parquet(
             fmd=meta,
             compression=compression,
             index_cols=index_cols,
-            **kwargs_pass
+            schema=schema,
+            **kwargs_pass,
         )
         for d, filename in zip(df.to_delayed(), filenames)
     ]
diff --git a/dask/dataframe/io/parquet/fastparquet.py b/dask/dataframe/io/parquet/fastparquet.py
--- a/dask/dataframe/io/parquet/fastparquet.py
+++ b/dask/dataframe/io/parquet/fastparquet.py
@@ -510,6 +510,7 @@ def initialize_write(
         partition_on=None,
         ignore_divisions=False,
         division_info=None,
+        schema=None,
         **kwargs
     ):
         if append and division_info is None:
@@ -579,7 +580,8 @@ def initialize_write(
             )
             i_offset = 0
 
-        return (fmd, i_offset)
+        schema = None  # ArrowEngine compatibility
+        return (fmd, schema, i_offset)
 
     @classmethod
     def write_partition(


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -960,12 +960,15 @@ def test_to_parquet_pyarrow_w_inconsistent_schema_by_partition_fails_by_default(
     )
 
     ddf = dd.from_pandas(df, npartitions=2)
-    # Note: `append_row_groups` will fail with pyarrow>17.1 for _metadata write
+    # In order to allow pyarrow to write an inconsistent schema,
+    # we need to avoid writing the _metadata file (will fail >0.17.1)
+    # and need to avoid schema inference (i.e. use `schema=None`)
     ddf.to_parquet(
         str(tmpdir),
         engine="pyarrow",
         partition_on=["partition_column"],
         write_metadata_file=False,
+        schema=None,
     )
 
     # Test that read fails because of default behavior when schema not provided
@@ -1069,6 +1072,53 @@ def test_to_parquet_pyarrow_w_inconsistent_schema_by_partition_succeeds_w_manual
     assert np.array_equal(ddf_after_write.partition_column, df.partition_column)
 
 
+@pytest.mark.parametrize("index", [False, True])
+@pytest.mark.parametrize(
+    "schema", ["infer", {"index": pa.string(), "amount": pa.int64()}]
+)
+def test_pyarrow_schema_inference(tmpdir, index, engine, schema):
+
+    check_pyarrow()
+    if pa.__version__ < LooseVersion("0.15.0"):
+        pytest.skip("PyArrow>=0.15 Required.")
+
+    tmpdir = str(tmpdir)
+    df = pd.DataFrame(
+        {
+            "index": ["1", "2", "3", "2", "3", "1", "4"],
+            "date": pd.to_datetime(
+                [
+                    "2017-01-01",
+                    "2017-01-01",
+                    "2017-01-01",
+                    "2017-01-02",
+                    "2017-01-02",
+                    "2017-01-06",
+                    "2017-01-09",
+                ]
+            ),
+            "amount": [100, 200, 300, 400, 500, 600, 700],
+        },
+        index=range(7, 14),
+    )
+    if index:
+        df = dd.from_pandas(df, npartitions=2).set_index("index")
+    else:
+        df = dd.from_pandas(df, npartitions=2)
+
+    df.to_parquet(tmpdir, engine="pyarrow", schema=schema, compute=False).compute(
+        scheduler="synchronous"
+    )
+    df_out = dd.read_parquet(tmpdir, engine=engine)
+
+    if index and engine == "fastparquet":
+        # Fastparquet not handling divisions for
+        # pyarrow-written dataset with string index
+        assert_eq(df, df_out, check_divisions=False)
+    else:
+        assert_eq(df, df_out)
+
+
 def test_partition_on(tmpdir, engine):
     tmpdir = str(tmpdir)
     df = pd.DataFrame(
@@ -2029,7 +2079,16 @@ def test_read_dir_nometa(tmpdir, write_engine, read_engine, statistics, remove_c
     assert_eq(ddf, ddf2, check_divisions=False)
 
 
-def test_timeseries_nulls_in_schema(tmpdir, engine):
+@pytest.mark.parametrize("schema", ["infer", None])
+def test_timeseries_nulls_in_schema(tmpdir, engine, schema):
+
+    if (
+        schema == "infer"
+        and engine == "pyarrow"
+        and pa.__version__ < LooseVersion("0.15.0")
+    ):
+        pytest.skip("PyArrow>=0.15 Required.")
+
     # GH#5608: relative path failing _metadata/_common_metadata detection.
     tmp_path = str(tmpdir.mkdir("files"))
     tmp_path = os.path.join(tmp_path, "../", "files")
@@ -2043,7 +2102,7 @@ def test_timeseries_nulls_in_schema(tmpdir, engine):
     ddf2.name = ddf2.name.where(ddf2.timestamp == "2000-01-01", None)
 
     # Note: `append_row_groups` will fail with pyarrow>0.17.1 for _metadata write
-    ddf2.to_parquet(tmp_path, engine=engine, write_metadata_file=False)
+    ddf2.to_parquet(tmp_path, engine=engine, write_metadata_file=False, schema=schema)
     ddf_read = dd.read_parquet(
         tmp_path, engine=engine, dataset={"validate_schema": False}
     )
@@ -2051,8 +2110,9 @@ def test_timeseries_nulls_in_schema(tmpdir, engine):
     assert_eq(ddf_read, ddf2, check_divisions=False, check_index=False)
 
     # Can force schema validation on each partition in pyarrow
-    if engine == "pyarrow":
-        # The schema mismatch should raise an error
+    if engine == "pyarrow" and schema is None:
+        # The schema mismatch should raise an error if the
+        # dataset was written with `schema=None` (no inference)
         with pytest.raises(ValueError):
             ddf_read = dd.read_parquet(
                 tmp_path, dataset={"validate_schema": True}, engine=engine