Repository: dask/dask

Issue:
dask.concat seems to change a DateTimeIndex dataframe in a way incompatible with fastparquet
Verified on Dask 0.16.0/0.15.4 

Here are two parquet files needed to replicate this
[Archive.zip](https://github.com/dask/dask/files/1511837/Archive.zip)
Code
```
import dask.dataframe as ddf
b2 = ddf.read_parquet('b2_small')
b3 = ddf.read_parquet('b3_small')
b2.to_parquet('b2b2b2')
b3.to_parquet('b3b3b3')
df = ddf.concat([b2, b3])
df.to_parquet('test')
```

Results in fastparquet throwing an exception that datetime64[ns] is not a supported datatype, but having gone through it with @martindurant it does not appear to be a fastparquet issue, and seems to be something that occurs because of the concat operation.


Discussion:
It would be great if you could generate the data in-line, and show whether the problem occurs because of concat (i.e., do a `.compute()`) or because of writing to parquet.
df.compute() works after the concat, it's only when I try to output to parquet that causes the problem. Using fastparquet.write directly to output the resultant pandas dataframe also works fine. 

For generating the data in-line for that I'd have to understand what about my data seems to be causing the issue, I have concatted other time indexed dataframes together and to_parquet has worked fine there.
Working from @shughes-uk 's example I've simplified this further to the following minimal example:

```python
import pandas as pd
import dask.dataframe as dd
b2 = pd.DataFrame({'x': ['a']}, index=pd.DatetimeIndex(['2015-03-24 00:00:16'], dtype='datetime64[ns]'))
b3 = pd.DataFrame({'x': ['c']}, index=pd.DatetimeIndex(['2015-03-29 00:00:44'], dtype='datetime64[ns]'))
b2['x'] = b2.x.astype('category')
b3['x'] = b3.x.astype('category')
db2 = dd.from_pandas(b2, npartitions=1)
db3 = dd.from_pandas(b3, npartitions=1)
df = dd.concat([db2, db3])
df.to_parquet('test')
```

It looks like we need a categorical, a datetime index, and a concat call.
Does this help to identify the issue @martindurant ?  

To be clear any of the following will cause the test to pass

1.  Dropping the categorization of x
2.  Moving the datetime index to a datetime column before calling dd.from_pandas
3.  Writing a single one of the dataframes
I have not looked in detail yet. That the dtype of a column should affect writing of the index is weird.
```python
/home/mrocklin/workspace/dask/dask/dataframe/io/parquet.py in _write_partition_fastparquet(df, fs, path, filename, fmd, compression, partition_on)
    170         with fs.open(fs.sep.join([path, filename]), 'wb') as fil:
    171             rgs = make_part_file(fil, df, fmd.schema, compression=compression,
--> 172                                  fmd=fmd)
    173     return rgs
    174 

/home/mrocklin/workspace/fastparquet/fastparquet/writer.py in make_part_file(f, data, schema, compression, fmd)
    604     with f as f:
    605         f.write(MARKER)
--> 606         rg = make_row_group(f, data, schema, compression=compression)
    607         if fmd is None:
    608             fmd = parquet_thrift.FileMetaData(num_rows=len(data),

/home/mrocklin/workspace/fastparquet/fastparquet/writer.py in make_row_group(f, data, schema, compression)
    592                 comp = compression
    593             chunk = write_column(f, data[column.name], column,
--> 594                                  compression=comp)
    595             rg.columns.append(chunk)
    596     rg.total_byte_size = sum([c.meta_data.total_uncompressed_size for c in

/home/mrocklin/workspace/fastparquet/fastparquet/writer.py in write_column(f, data, selement, compression)
    498     start = f.tell()
    499     bdata = definition_data + repetition_data + encode[encoding](
--> 500             data, selement)
    501     bdata += 8 * b'\x00'
    502     try:

/home/mrocklin/workspace/fastparquet/fastparquet/writer.py in encode_plain(data, se)
    243 def encode_plain(data, se):
    244     """PLAIN encoding; returns byte representation"""
--> 245     out = convert(data, se)
    246     if se.type == parquet_thrift.Type.BYTE_ARRAY:
    247         return pack_byte_array(list(out))

/home/mrocklin/workspace/fastparquet/fastparquet/writer.py in convert(data, se)
    203         out['day'] = day
    204     else:
--> 205         raise ValueError("Don't know how to convert data type: %s" % dtype)
    206     return out
    207 

ValueError: Don't know how to convert data type: datetime64[ns]
```
@mrocklin poking around with a bit of guidance from @martindurant, the index ends up as int64 after the concat, which is unexpected.

Resolving patch:
diff --git a/dask/dataframe/methods.py b/dask/dataframe/methods.py
--- a/dask/dataframe/methods.py
+++ b/dask/dataframe/methods.py
@@ -259,9 +259,13 @@ def concat(dfs, axis=0, join='outer', uniform=False):
 
     # Handle categorical index separately
     dfs0_index = dfs[0].index
-    if (isinstance(dfs0_index, pd.CategoricalIndex) or
-            (isinstance(dfs0_index, pd.MultiIndex) and
-             any(isinstance(i, pd.CategoricalIndex) for i in dfs0_index.levels))):
+
+    has_categoricalindex = (
+        isinstance(dfs0_index, pd.CategoricalIndex) or
+        (isinstance(dfs0_index, pd.MultiIndex) and
+         any(isinstance(i, pd.CategoricalIndex) for i in dfs0_index.levels)))
+
+    if has_categoricalindex:
         dfs2 = [df.reset_index(drop=True) for df in dfs]
         ind = concat([df.index for df in dfs])
     else:
@@ -290,6 +294,7 @@ def concat(dfs, axis=0, join='outer', uniform=False):
             not_cat = cat_mask[~cat_mask].index
             out = pd.concat([df[df.columns.intersection(not_cat)]
                              for df in dfs3], join=join)
+            temp_ind = out.index
             for col in cat_mask.index.difference(not_cat):
                 # Find an example of categoricals in this column
                 for df in dfs3:
@@ -308,6 +313,10 @@ def concat(dfs, axis=0, join='outer', uniform=False):
                                                          sample.cat.ordered)
                         parts.append(data)
                 out[col] = union_categoricals(parts)
+                # Pandas resets index type on assignment if frame is empty
+                # https://github.com/pandas-dev/pandas/issues/17101
+                if not len(temp_ind):
+                    out.index = temp_ind
             out = out.reindex(columns=cat_mask.index)
         else:
             # pandas may raise a RuntimeWarning for comparing ints and strs
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -523,6 +523,14 @@ def concat(dfs, axis=0, join='outer', interleave_partitions=False):
         Whether to concatenate DataFrames ignoring its order. If True, every
         divisions are concatenated each by each.
 
+    Notes
+    -----
+    This differs in from ``pd.concat`` in the when concatenating Categoricals
+    with different categories. Pandas currently coerces those to objects
+    before concatenating. Coercing to objects is very expensive for large
+    arrays, so dask preserves the Categoricals by taking the union of
+    the categories.
+
     Examples
     --------
 
@@ -558,6 +566,14 @@ def concat(dfs, axis=0, join='outer', interleave_partitions=False):
     dd.DataFrame<y, divisions=(1, 4, 10)>
     >>> dd.concat([a, b])                               # doctest: +SKIP
     dd.DataFrame<concat-..., divisions=(None, None, None, None)>
+
+    Different categoricals are unioned
+
+    >> dd.concat([                                     # doctest: +SKIP
+    ...     dd.from_pandas(pd.Series(['a', 'b'], dtype='category'), 1),
+    ...     dd.from_pandas(pd.Series(['a', 'c'], dtype='category'), 1),
+    ... ], interleave_partitions=True).dtype
+    CategoricalDtype(categories=['a', 'b', 'c'], ordered=False)
     """
     if not isinstance(dfs, list):
         raise TypeError("dfs must be a list of DataFrames/Series objects")


Test patch:
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -1072,6 +1072,29 @@ def check_and_return(ddfs, dfs, join):
         assert has_known_categories(res.y) == known
 
 
+def test_concat_datetimeindex():
+    # https://github.com/dask/dask/issues/2932
+    b2 = pd.DataFrame({'x': ['a']},
+                      index=pd.DatetimeIndex(['2015-03-24 00:00:16'],
+                                             dtype='datetime64[ns]'))
+    b3 = pd.DataFrame({'x': ['c']},
+                      index=pd.DatetimeIndex(['2015-03-29 00:00:44'],
+                                             dtype='datetime64[ns]'))
+
+    b2['x'] = b2.x.astype('category').cat.set_categories(['a', 'c'])
+    b3['x'] = b3.x.astype('category').cat.set_categories(['a', 'c'])
+
+    db2 = dd.from_pandas(b2, 1)
+    db3 = dd.from_pandas(b3, 1)
+
+    result = concat([b2.iloc[:0], b3.iloc[:0]])
+    assert result.index.dtype == '<M8[ns]'
+
+    result = dd.concat([db2, db3])
+    expected = pd.concat([b2, b3])
+    assert_eq(result, expected)
+
+
 def test_append():
     df = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],
                        'b': [1, 2, 3, 4, 5, 6]})