Repository: dask/dask

Issue:
Exception while providing a categorical dictionary meta
There is an exception when forcing *name* to be an unknown category instead of a string object dtype.

```python
import dask
dd = dask.datasets.timeseries()
dd = dd.map_partitions(lambda part: part, meta={'id': 'int32', 'name': 'str', 'x': 'float64', 'y': 'float64'})      # Ok
dd = dd.map_partitions(lambda part: part, meta={'id': 'int32', 'name': 'category', 'x': 'float64', 'y': 'float64'}) # Raise
```
![image](https://user-images.githubusercontent.com/1694892/179756928-b7e9b03a-1863-48e4-9fbb-805c613cec72.png)



**Environment**:

- Dask version:  2022.7.0
- Python version: 3.9
- Operating System: Window 10
- Install method (conda, pip, source): pip



Discussion:
Thanks for the reproducer, @epizut! Adding a full traceback for reference:

<details open>
<summary> traceback</summary>
<br>

```python
In [1]: import dask.datasets

In [2]: df = dask.datasets.timeseries()

In [3]: df = df.map_partitions(
   ...:     lambda part: part,
   ...:     meta={"id": "int32", "name": "category", "x": "float64", "y": "float64"},
   ...: )
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
File ~/software/repos/dask/dask/dataframe/dispatch.py:128, in make_meta(x, index, parent_meta)
    127 try:
--> 128     return make_meta_dispatch(x, index=index)
    129 except TypeError:

File ~/software/repos/dask/dask/utils.py:606, in Dispatch.__call__(self, arg, *args, **kwargs)
    603 """
    604 Call the corresponding method based on type of argument.
    605 """
--> 606 meth = self.dispatch(type(arg))
    607 return meth(arg, *args, **kwargs)

File ~/software/repos/dask/dask/utils.py:600, in Dispatch.dispatch(self, cls)
    599         return self.dispatch(cls)  # recurse
--> 600 raise TypeError(f"No dispatch for {cls}")

TypeError: No dispatch for <class 'dict'>

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
Input In [3], in <cell line: 1>()
----> 1 df = df.map_partitions(
      2     lambda part: part,
      3     meta={"id": "int32", "name": "category", "x": "float64", "y": "float64"},
      4 )

File ~/software/repos/dask/dask/dataframe/core.py:866, in _Frame.map_partitions(self, func, *args, **kwargs)
    739 @insert_meta_param_description(pad=12)
    740 def map_partitions(self, func, *args, **kwargs):
    741     """Apply Python function on each DataFrame partition.
    742 
    743     Note that the index and divisions are assumed to remain unchanged.
   (...)
    864     None as the division.
    865     """
--> 866     return map_partitions(func, self, *args, **kwargs)

File ~/software/repos/dask/dask/dataframe/core.py:6475, in map_partitions(func, meta, enforce_metadata, transform_divisions, align_dataframes, *args, **kwargs)
   6468         raise ValueError(
   6469             f"{e}. If you don't want the partitions to be aligned, and are "
   6470             "calling `map_partitions` directly, pass `align_dataframes=False`."
   6471         ) from e
   6473 dfs = [df for df in args if isinstance(df, _Frame)]
-> 6475 meta = _get_meta_map_partitions(args, dfs, func, kwargs, meta, parent_meta)
   6477 if all(isinstance(arg, Scalar) for arg in args):
   6478     layer = {
   6479         (name, 0): (
   6480             apply,
   (...)
   6484         )
   6485     }

File ~/software/repos/dask/dask/dataframe/core.py:6590, in _get_meta_map_partitions(args, dfs, func, kwargs, meta, parent_meta)
   6588     meta_is_emulated = True
   6589 else:
-> 6590     meta = make_meta(meta, index=meta_index, parent_meta=parent_meta)
   6591     meta_is_emulated = False
   6593 if not (has_parallel_type(meta) or is_arraylike(meta) and meta.shape) and not all(
   6594     isinstance(arg, Scalar) for arg in args
   6595 ):

File ~/software/repos/dask/dask/dataframe/dispatch.py:132, in make_meta(x, index, parent_meta)
    130 if parent_meta is not None:
    131     func = make_meta_obj.dispatch(type(parent_meta))
--> 132     return func(x, index=index)
    133 else:
    134     # Default to using the pandas backend
    135     # if ``parent_meta`` is not specified
    136     func = make_meta_obj.dispatch(pd.DataFrame)

File ~/software/repos/dask/dask/dataframe/backends.py:135, in make_meta_object(x, index)
    131     index = make_meta_dispatch(index)
    133 if isinstance(x, dict):
    134     return pd.DataFrame(
--> 135         {c: _empty_series(c, d, index=index) for (c, d) in x.items()}, index=index
    136     )
    137 if isinstance(x, tuple) and len(x) == 2:
    138     return _empty_series(x[0], x[1], index=index)

File ~/software/repos/dask/dask/dataframe/backends.py:135, in <dictcomp>(.0)
    131     index = make_meta_dispatch(index)
    133 if isinstance(x, dict):
    134     return pd.DataFrame(
--> 135         {c: _empty_series(c, d, index=index) for (c, d) in x.items()}, index=index
    136     )
    137 if isinstance(x, tuple) and len(x) == 2:
    138     return _empty_series(x[0], x[1], index=index)

File ~/software/repos/dask/dask/dataframe/utils.py:288, in _empty_series(name, dtype, index)
    286 def _empty_series(name, dtype, index=None):
    287     if isinstance(dtype, str) and dtype == "category":
--> 288         return pd.Series(
    289             pd.Categorical([UNKNOWN_CATEGORIES]), name=name, index=index
    290         ).iloc[:0]
    291     return pd.Series([], dtype=dtype, name=name, index=index)

File ~/.pyenv/versions/3.10.4/envs/dask-dev/lib/python3.10/site-packages/pandas/core/series.py:442, in Series.__init__(self, data, index, dtype, name, copy, fastpath)
    440     index = default_index(len(data))
    441 elif is_list_like(data):
--> 442     com.require_length_match(data, index)
    444 # create/copy the manager
    445 if isinstance(data, (SingleBlockManager, SingleArrayManager)):

File ~/.pyenv/versions/3.10.4/envs/dask-dev/lib/python3.10/site-packages/pandas/core/common.py:557, in require_length_match(data, index)
    553 """
    554 Check the length of data matches the length of the index.
    555 """
    556 if len(data) != len(index):
--> 557     raise ValueError(
    558         "Length of values "
    559         f"({len(data)}) "
    560         "does not match length of index "
    561         f"({len(index)})"
    562     )

ValueError: Length of values (1) does not match length of index (0)
```
</details>
This issue can be reproduced with a shorted snippet:
```python
dask.dataframe.utils.make_meta({'id': 'int32', 'name': 'category', 'x': 'float64', 'y': 'float64'}, index=pd.DatetimeIndex(['1970-01-01', 1970-01-02]))
```
@ncclementi and I are still looking into this, here are some observations.

The issue seems to be in how we're creating the Series on line 289 here:
https://github.com/dask/dask/blob/1f0af935aa5ac04bb9ca702d529cd3ae59280843/dask/dataframe/utils.py#L286-L291

The index seems to always have length=0, while it should be 1 because `pd.Categorical([UNKNOWN_CATEGORIES])` will always create a Categorical object of length 1. If we don't provide index (i.e., `index=None`), this works. :/

@jsignell Do you have thoughts on how we should proceed?
Sorry for the slow response. I wonder if we can just set the index _after_ we subset the series? Something like:

```python
 def _empty_series(name, dtype, index=None): 
     if isinstance(dtype, str) and dtype == "category": 
         s = pd.Series( 
             pd.Categorical([UNKNOWN_CATEGORIES]), name=name 
         ).iloc[:0] 
        s.index = index
        return s
     return pd.Series([], dtype=dtype, name=name, index=index) 

Resolving patch:
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -298,9 +298,10 @@ def clear_known_categories(x, cols=None, index=True):
 
 def _empty_series(name, dtype, index=None):
     if isinstance(dtype, str) and dtype == "category":
-        return pd.Series(
-            pd.Categorical([UNKNOWN_CATEGORIES]), name=name, index=index
-        ).iloc[:0]
+        s = pd.Series(pd.Categorical([UNKNOWN_CATEGORIES]), name=name).iloc[:0]
+        if index is not None:
+            s.index = make_meta(index)
+        return s
     return pd.Series([], dtype=dtype, name=name, index=index)
 
 


Test patch:
diff --git a/dask/dataframe/tests/test_utils_dataframe.py b/dask/dataframe/tests/test_utils_dataframe.py
--- a/dask/dataframe/tests/test_utils_dataframe.py
+++ b/dask/dataframe/tests/test_utils_dataframe.py
@@ -133,6 +133,12 @@ def __next__(self):
     assert len(meta.cat.categories) == 1
     assert meta.cat.categories[0] == UNKNOWN_CATEGORIES
 
+    # Categorials with Index
+    meta = make_meta({"a": "category", "b": "int64"}, index=idx)
+    assert len(meta.a.cat.categories) == 1
+    assert meta.index.dtype == "int64"
+    assert meta.index.empty
+
     # Numpy scalar
     meta = make_meta(np.float64(1.0), parent_meta=df)
     assert isinstance(meta, np.float64)