Repository: dask/dask

Issue:
Meta not updated correctly with MultiIndex while using Accessor functions in DataFrame
Consider the following case of DF:
```python
ddf = dd.from_pandas(pd.DataFrame(['<t1><t2>', '<t2><t3>'], columns=['tags']), npartitions=1)
ddf.head()
```
```
       tags
0  <t1><t2>
1  <t2><t3>
```

Using `str.extractall` function works fine, see:
```python
t = ddf.tags.str.extractall('<([^>]*)>')\
    .rename(columns={0: 'tag'})
print(t.head()
```
```
        tag
  match    
0 0      t1
  1      t2
1 0      t2
  1      t3
```
But the meta is not updated:
```python
print(t._meta.index)
```
```
Index([], dtype='object')
```

Thus, operations like `reset_index` fail as follows:
```python
t.reset_index().head()
```
```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-100-e6186d78fb03> in <module>()
----> 1 t.reset_index().head()

C:\ProgramData\Anaconda3\lib\site-packages\dask\dataframe\core.py in head(self, n, npartitions, compute)
    844 
    845         if compute:
--> 846             result = result.compute()
    847         return result
    848 

C:\ProgramData\Anaconda3\lib\site-packages\dask\base.py in compute(self, **kwargs)
    133         dask.base.compute
    134         """
--> 135         (result,) = compute(self, traverse=False, **kwargs)
    136         return result
    137 

C:\ProgramData\Anaconda3\lib\site-packages\dask\base.py in compute(*args, **kwargs)
    331     postcomputes = [a.__dask_postcompute__() if is_dask_collection(a)
    332                     else (None, a) for a in args]
--> 333     results = get(dsk, keys, **kwargs)
    334     results_iter = iter(results)
    335     return tuple(a if f is None else f(next(results_iter), *a)

C:\ProgramData\Anaconda3\lib\site-packages\dask\multiprocessing.py in get(dsk, keys, num_workers, func_loads, func_dumps, optimize_graph, **kwargs)
    175                            get_id=_process_get_id, dumps=dumps, loads=loads,
    176                            pack_exception=pack_exception,
--> 177                            raise_exception=reraise, **kwargs)
    178     finally:
    179         if cleanup:

C:\ProgramData\Anaconda3\lib\site-packages\dask\local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    519                         _execute_task(task, data)  # Re-execute locally
    520                     else:
--> 521                         raise_exception(exc, tb)
    522                 res, worker_id = loads(res_info)
    523                 state['cache'][key] = res

C:\ProgramData\Anaconda3\lib\site-packages\dask\compatibility.py in reraise(exc, tb)
     57     def reraise(exc, tb=None):
     58         if exc.__traceback__ is not tb:
---> 59             raise exc.with_traceback(tb)
     60         raise exc
     61 

C:\ProgramData\Anaconda3\lib\site-packages\dask\local.py in execute_task()
    288     try:
    289         task, data = loads(task_info)
--> 290         result = _execute_task(task, data)
    291         id = get_id()
    292         result = dumps((result, id))

C:\ProgramData\Anaconda3\lib\site-packages\dask\local.py in _execute_task()
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

C:\ProgramData\Anaconda3\lib\site-packages\dask\local.py in <listcomp>()
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

C:\ProgramData\Anaconda3\lib\site-packages\dask\local.py in _execute_task()
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

C:\ProgramData\Anaconda3\lib\site-packages\dask\dataframe\core.py in apply_and_enforce()
   3268             return meta
   3269         c = meta.columns if isinstance(df, pd.DataFrame) else meta.name
-> 3270         return _rename(c, df)
   3271     return df
   3272 

C:\ProgramData\Anaconda3\lib\site-packages\dask\dataframe\core.py in _rename()
   3305         # deep=False doesn't doesn't copy any data/indices, so this is cheap
   3306         df = df.copy(deep=False)
-> 3307         df.columns = columns
   3308         return df
   3309     elif isinstance(df, (pd.Series, pd.Index)):

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\generic.py in __setattr__()
   3625         try:
   3626             object.__getattribute__(self, name)
-> 3627             return object.__setattr__(self, name, value)
   3628         except AttributeError:
   3629             pass

pandas/_libs/properties.pyx in pandas._libs.properties.AxisProperty.__set__()

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\generic.py in _set_axis()
    557 
    558     def _set_axis(self, axis, labels):
--> 559         self._data.set_axis(axis, labels)
    560         self._clear_item_cache()
    561 

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py in set_axis()
   3067             raise ValueError('Length mismatch: Expected axis has %d elements, '
   3068                              'new values have %d elements' %
-> 3069                              (old_len, new_len))
   3070 
   3071         self.axes[axis] = new_labels

ValueError: Length mismatch: Expected axis has 3 elements, new values have 2 elements
```
Following suggestion in #2929, as the issue seems to be similar, after updating meta by hand `reset_index` works:
```python
t._meta = pd.DataFrame(columns=['tag'], dtype=str, index=pd.MultiIndex([[], []], [[], []], names=[None, 'match']))
print(t._meta.index)
```
```
MultiIndex(levels=[[], []],
           labels=[[], []],
           names=[None, 'match'])
```
```python
print(t.reset_index().head())
```
```
   level_0  match tag
0        0      0  t1
1        0      1  t2
2        1      0  t2
3        1      1  t3
```
IMO this should be done withing the accessor, as doing it manually is a bit cumbersome. It seems to only be related to the functions that change index, but for `str` I only recall `extractall`.

Versions:
* dask: 0.16.0
* python: 3.6  


Discussion:
Thanks for the report.

I think this is a bug in pandas not having the correct index on a `.str.extractall` for an empty result (no matches or empty input). I opened https://github.com/pandas-dev/pandas/issues/19034 to track that if you're interested in following along (and maybe submitting a fix?)
Thanks. I'll look into it and see if I can find and/or fix anything.
@TomAugspurger I've done some manual tests with the PR pandas-dev/pandas#19075 and it seems to fix this issue. Do you think we need extra tests for that or just flag this as fixed with the upstream PR?

Resolving patch:
diff --git a/dask/dataframe/accessor.py b/dask/dataframe/accessor.py
--- a/dask/dataframe/accessor.py
+++ b/dask/dataframe/accessor.py
@@ -5,7 +5,7 @@
 from toolz import partial
 
 from ..utils import derived_from
-from .utils import is_categorical_dtype
+from .utils import is_categorical_dtype, PANDAS_VERSION
 
 
 def maybe_wrap_pandas(obj, x):
@@ -59,7 +59,7 @@ def _property_map(self, attr):
 
     def _function_map(self, attr, *args, **kwargs):
         meta = self._delegate_method(self._series._meta_nonempty,
-                                     self._accessor_name,  attr, args, kwargs)
+                                     self._accessor_name, attr, args, kwargs)
         token = '%s-%s' % (self._accessor_name, attr)
         return self._series.map_partitions(self._delegate_method,
                                            self._accessor_name, attr, args,
@@ -134,11 +134,27 @@ def cat(self, others=None, sep=None, na_rep=None):
         return self._series.map_partitions(str_cat, *others, sep=sep,
                                            na_rep=na_rep, meta=self._series._meta)
 
+    @derived_from(pd.core.strings.StringMethods)
+    def extractall(self, pat, flags=0):
+        # TODO: metadata inference here won't be necessary for pandas >= 0.23.0
+        meta = self._series._meta.str.extractall(pat, flags=flags)
+        if PANDAS_VERSION < '0.23.0':
+            index_name = self._series.index.name
+            meta.index = pd.MultiIndex(levels=[[], []],
+                                       labels=[[], []],
+                                       names=[index_name, 'match'])
+        return self._series.map_partitions(str_extractall, pat, flags,
+                                           meta=meta, token='str-extractall')
+
     def __getitem__(self, index):
         return self._series.map_partitions(str_get, index,
                                            meta=self._series._meta)
 
 
+def str_extractall(series, pat, flags):
+    return series.str.extractall(pat, flags=flags)
+
+
 def str_get(series, index):
     """ Implements series.str[index] """
     return series.str[index]


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1476,6 +1476,10 @@ def test_str_accessor():
     assert_eq(ddf.x.str[:2], df.x.str[:2])
     assert_eq(ddf.x.str[1], df.x.str[1])
 
+    # str.extractall
+    assert_eq(ddf.x.str.extractall('(.*)b(.*)'),
+              df.x.str.extractall('(.*)b(.*)'))
+
     # str.cat
     sol = df.x.str.cat(df.x.str.upper(), sep=':')
     assert_eq(ddf.x.str.cat(ddf.x.str.upper(), sep=':'), sol)