Repository: dask/dask

Issue:
Issue with parquet and categories
I'm not 100% sure if this is a bug or me just not knowing how to use dask, but I'm getting the following bug,  Note the "astype('category')" seems to be the trigger here; pdf['y'] = pdf['y'].astype('f8') does not trigger the problem.


```python
import pandas as pd
import dask.dataframe as da

pd.__version__
# '0.19.2'
import dask
dask.__version__
# '0.13.0'

df = pd.DataFrame({'ID': ['a', 'b', 'a'],'y':[1,2,3]},index=None)
ddf = da.from_pandas(df,npartitions=2)

ddf.to_parquet("/tmp/bug.parq/")

pdf = da.read_parquet('/tmp/bug.parq/')

pdf['ID'] = pdf['ID'].astype('category')
# pdf['y'] = pdf['y'] * 20

pdf.head(2)

pdf.to_parquet('/tmp/bug-write.parq/')
```

Other than this it's running python3.5 in a jupyter web notebook.

This gives me:

```
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/utils.py in raise_on_meta_error(funcname)
    153     try:
--> 154         yield
    155     except Exception as e:

[ ...  lots of trace ... ]

ValueError: Metadata inference failed in `reset_index`.

Original error is below:
------------------------
IndexError('index 0 is out of bounds for axis 0 with size 0',)

Traceback:
---------
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/utils.py", line 154, in raise_on_meta_error
    yield
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/core.py", line 2952, in _emulate
    return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/core.py", line 2936, in _extract_meta
    return tuple([_extract_meta(_x, nonempty) for _x in x])
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/core.py", line 2936, in <listcomp>
    return tuple([_extract_meta(_x, nonempty) for _x in x])
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/core.py", line 2932, in _extract_meta
    return x._meta_nonempty if nonempty else x._meta
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/core.py", line 256, in _meta_nonempty
    return meta_nonempty(self._meta)
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/utils.py", line 339, in meta_nonempty
    for i, c in enumerate(x.columns)}
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/utils.py", line 339, in <dictcomp>
    for i, c in enumerate(x.columns)}
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/dask/dataframe/utils.py", line 314, in _nonempty_series
    entry = s.cat.categories[0]
  File "/home/pcaldon/ruleopt/venv/lib/python3.5/site-packages/pandas/indexes/base.py", line 1423, in __getitem__
    return getitem(key)


```



Discussion:

Resolving patch:
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -251,9 +251,13 @@ def _nonempty_index(idx):
         return pd.TimedeltaIndex(data, start=start, periods=2, freq=idx.freq,
                                  name=idx.name)
     elif typ is pd.CategoricalIndex:
-        element = idx.categories[0]
-        return pd.CategoricalIndex([element, element],
-                                   categories=idx.categories,
+        if len(idx.categories):
+            data = [idx.categories[0]] * 2
+            cats = idx.categories
+        else:
+            data = _nonempty_index(idx.categories)
+            cats = data.unique()
+        return pd.CategoricalIndex(data, categories=cats,
                                    ordered=idx.ordered, name=idx.name)
     elif typ is pd.MultiIndex:
         levels = [_nonempty_index(i) for i in idx.levels]
@@ -311,9 +315,13 @@ def _nonempty_series(s, idx):
         entry = pd.Timestamp('1970-01-01', tz=dtype.tz)
         data = [entry, entry]
     elif is_categorical_dtype(dtype):
-        entry = s.cat.categories[0]
-        data = pd.Categorical([entry, entry],
-                              categories=s.cat.categories,
+        if len(s.cat.categories):
+            data = [s.cat.categories[0]] * 2
+            cats = s.cat.categories
+        else:
+            data = _nonempty_index(s.cat.categories)
+            cats = data.unique()
+        data = pd.Categorical(data, categories=cats,
                               ordered=s.cat.ordered)
     else:
         entry = _scalar_from_dtype(dtype)


Test patch:
diff --git a/dask/dataframe/tests/test_utils_dataframe.py b/dask/dataframe/tests/test_utils_dataframe.py
--- a/dask/dataframe/tests/test_utils_dataframe.py
+++ b/dask/dataframe/tests/test_utils_dataframe.py
@@ -139,6 +139,25 @@ def test_meta_duplicated():
     tm.assert_frame_equal(res, exp)
 
 
+def test_meta_nonempty_empty_categories():
+    for dtype in ['O', 'f8', 'M8']:
+        # Index
+        idx = pd.CategoricalIndex([], pd.Index([], dtype=dtype),
+                                  ordered=True, name='foo')
+        res = meta_nonempty(idx)
+        assert type(res) is pd.CategoricalIndex
+        assert type(res.categories) is type(idx.categories)
+        assert res.ordered == idx.ordered
+        assert res.name == idx.name
+        # Series
+        s = idx.to_series()
+        res = meta_nonempty(s)
+        assert res.dtype == s.dtype
+        assert type(res.cat.categories) is type(s.cat.categories)
+        assert res.cat.ordered == s.cat.ordered
+        assert res.name == s.name
+
+
 def test_meta_nonempty_index():
     idx = pd.RangeIndex(1, name='foo')
     res = meta_nonempty(idx)