Repository: dask/dask

Issue:
Dataframe.reset_index() fails when index is named
```
import numpy as np
import pandas as pd
import dask.dataframe as dd


def test(idx_name=None):
    index = pd.Series(list('abcde'), name=idx_name)
    df = pd.DataFrame({
        'A': np.arange(5, dtype=np.int32),
        'B': np.arange(10, 15, dtype=np.int32),
    }, index=index)
    df = dd.from_pandas(df, npartitions=2)
    df = df.map_partitions(lambda p: (1.0 / p.sum(axis=1)).astype(np.float32),
                           meta=('new', np.float32))
    df = df.reset_index()
    df = df.compute()
    return df


if __name__ == '__main__':
    for idx_name in None, 'index', 'foo':
        print("\nidx_name: {!r}".format(idx_name))
        try:
            print(test(idx_name))
        except ValueError as ex:
            print(ex)
```
```

idx_name: None
  index       new
0     a  0.100000
1     b  0.083333
2     c  0.071429
0     d  0.062500
1     e  0.055556

idx_name: 'index'
  index       new
0     a  0.100000
1     b  0.083333
2     c  0.071429
0     d  0.062500
1     e  0.055556

idx_name: 'foo'
The columns in the computed data do not match the columns in the provided metadata
```


Discussion:
If I remove your meta= declaration then things are fine.  It's finding that
the declared metadata is diffferent from the actual results.  In this case
that's probably because you're not specifying the index name in the
metadata but one exists.  If you need to pass meta= then I recommend
passing meta= as a full Series with the index name.

On Sat, Feb 2, 2019 at 2:13 AM George Sakkis <notifications@github.com>
wrote:

> import numpy as np
> import pandas as pd
> import dask.dataframe as dd
>
>
> def test(idx_name=None):
>     index = pd.Series(list('abcde'), name=idx_name)
>     df = pd.DataFrame({
>         'A': np.arange(5, dtype=np.int32),
>         'B': np.arange(10, 15, dtype=np.int32),
>     }, index=index)
>     df = dd.from_pandas(df, npartitions=2)
>     df = df.map_partitions(lambda p: (1.0 / p.sum(axis=1)).astype(np.float32),
>                            meta=('new', np.float32))
>     df = df.reset_index()
>     df = df.compute()
>     return df
>
>
> if __name__ == '__main__':
>     for idx_name in None, 'index', 'foo':
>         print("\nidx_name: {!r}".format(idx_name))
>         try:
>             print(test(idx_name))
>         except ValueError as ex:
>             print(ex)
>
>
> idx_name: None
>   index       new
> 0     a  0.100000
> 1     b  0.083333
> 2     c  0.071429
> 0     d  0.062500
> 1     e  0.055556
>
> idx_name: 'index'
>   index       new
> 0     a  0.100000
> 1     b  0.083333
> 2     c  0.071429
> 0     d  0.062500
> 1     e  0.055556
>
> idx_name: 'foo'
> The columns in the computed data do not match the columns in the provided metadata
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/4454>, or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AASszHJwdz_7GNk7HTVv3qq0vr5dCfAXks5vJWThgaJpZM4afhkF>
> .
>

Indeed, removing `meta` works in this trimmed down example but in the actual case I encountered the issue, it couldn't be inferred automatically.

>  If you need to pass meta= then I recommend passing meta= as a full Series with the index name.

After some trial and error, the thing that worked was `meta=pd.Series([], name='new', dtype=np.float32, index=df.index.head(0))`. If this is indeed the best/simplest way to specify it, I'd say there is room for improvement, if nothing else at least in the documentation about `meta`.
Usability improvements here would be welcome

On Sun, Feb 3, 2019 at 2:42 AM George Sakkis <notifications@github.com>
wrote:

> Indeed, removing meta works in this trimmed down example but in the
> actual case I encountered the issue, it couldn't be inferred automatically.
>
> If you need to pass meta= then I recommend passing meta= as a full Series
> with the index name.
>
> After some trial and error, the thing that worked was meta=pd.Series([],
> name='new', dtype=np.float32, index=df.index.head(0)). If this is indeed
> the best/simplest way to specify it, I'd say there is room for improvement,
> if nothing else at least in the documentation about meta.
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/4454#issuecomment-460040858>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszDIdvTot6kLh2_2xAHI9dt8yw4pOks5vJr0WgaJpZM4afhkF>
> .
>

@gsakkis do you have any specific recommendations on how to improve documentation here?  Otherwise I'm going to close this.
Documentation aside I have the feeling this case could be handled better by inferring the correct index name; will need to look into it and hopefully post a PR.
Glad to hear it!

On Tue, Feb 19, 2019 at 1:19 AM George Sakkis <notifications@github.com>
wrote:

> Documentation aside I have the feeling this case could be handled better
> by inferring the correct index name; will need to look into it and
> hopefully post a PR.
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/4454#issuecomment-465052453>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszE5-QnWaR78N0xMChLlZgXRrhSEpks5vO8G8gaJpZM4afhkF>
> .
>


Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -385,7 +385,8 @@ def __repr__(self):
     @property
     def index(self):
         """Return dask Index instance"""
-        return self.map_partitions(getattr, 'index', token=self._name + '-index')
+        return self.map_partitions(getattr, 'index', token=self._name + '-index',
+                                   meta=self._meta.index)
 
     def reset_index(self, drop=False):
         """Reset the index to the default index.
@@ -2128,7 +2129,7 @@ def map(self, arg, na_action=None, meta=no_default):
         if meta is no_default:
             meta = _emulate(M.map, self, arg, na_action=na_action, udf=True)
         else:
-            meta = make_meta(meta)
+            meta = make_meta(meta, index=getattr(make_meta(self), 'index', None))
 
         return Series(graph, name, meta, self.divisions)
 
@@ -3583,8 +3584,9 @@ def apply_concat_apply(args, chunk=None, aggregate=None, combine=None,
     if not isinstance(args, (tuple, list)):
         args = [args]
 
-    npartitions = set(arg.npartitions for arg in args
-                      if isinstance(arg, _Frame))
+    dfs = [arg for arg in args if isinstance(arg, _Frame)]
+
+    npartitions = set(arg.npartitions for arg in dfs)
     if len(npartitions) > 1:
         raise ValueError("All arguments must have same number of partitions")
     npartitions = npartitions.pop()
@@ -3610,13 +3612,13 @@ def apply_concat_apply(args, chunk=None, aggregate=None, combine=None,
         dsk = {(a, 0, i, 0): (apply, chunk,
                               [(x._name, i) if isinstance(x, _Frame)
                                else x for x in args], chunk_kwargs)
-               for i in range(args[0].npartitions)}
+               for i in range(npartitions)}
 
     # Split
     if split_out and split_out > 1:
         split_prefix = 'split-%s' % token_key
         shard_prefix = 'shard-%s' % token_key
-        for i in range(args[0].npartitions):
+        for i in range(npartitions):
             dsk[(split_prefix, i)] = (hash_shard, (a, 0, i, 0), split_out,
                                       split_out_setup, split_out_setup_kwargs)
             for j in range(split_out):
@@ -3654,11 +3656,10 @@ def apply_concat_apply(args, chunk=None, aggregate=None, combine=None,
         meta_chunk = _emulate(chunk, *args, udf=True, **chunk_kwargs)
         meta = _emulate(aggregate, _concat([meta_chunk]), udf=True,
                         **aggregate_kwargs)
-    meta = make_meta(meta)
-
-    dependencies = [arg for arg in args if isinstance(arg, _Frame)]
+    meta = make_meta(meta, index=(getattr(make_meta(dfs[0]), 'index', None)
+                                  if dfs else None))
 
-    graph = HighLevelGraph.from_collections(b, dsk, dependencies=dependencies)
+    graph = HighLevelGraph.from_collections(b, dsk, dependencies=dfs)
 
     divisions = [None] * (split_out + 1)
 
@@ -3713,9 +3714,6 @@ def map_partitions(func, *args, **kwargs):
     meta = kwargs.pop('meta', no_default)
     name = kwargs.pop('token', None)
 
-    if meta is not no_default:
-        meta = make_meta(meta)
-
     # Normalize keyword arguments
     kwargs2 = {k: normalize_arg(v) for k, v in kwargs.items()}
 
@@ -3730,9 +3728,13 @@ def map_partitions(func, *args, **kwargs):
     from .multi import _maybe_align_partitions
     args = _maybe_from_pandas(args)
     args = _maybe_align_partitions(args)
+    dfs = [df for df in args if isinstance(df, _Frame)]
+    meta_index = getattr(make_meta(dfs[0]), 'index', None) if dfs else None
 
     if meta is no_default:
         meta = _emulate(func, *args, udf=True, **kwargs2)
+    else:
+        meta = make_meta(meta, index=meta_index)
 
     if all(isinstance(arg, Scalar) for arg in args):
         layer = {(name, 0):
@@ -3742,7 +3744,9 @@ def map_partitions(func, *args, **kwargs):
     elif not (has_parallel_type(meta) or is_arraylike(meta)):
         # If `meta` is not a pandas object, the concatenated results will be a
         # different type
-        meta = _concat([meta])
+        meta = make_meta(_concat([meta]), index=meta_index)
+
+    # Ensure meta is empty series
     meta = make_meta(meta)
 
     args2 = []
@@ -3776,7 +3780,6 @@ def map_partitions(func, *args, **kwargs):
         _meta=meta,
         **kwargs3
     )
-    dfs = [df for df in args if isinstance(df, _Frame)]
 
     graph = HighLevelGraph.from_collections(name, dsk, dependencies=dependencies)
     return new_dd_object(graph, name, meta, dfs[0].divisions)
diff --git a/dask/dataframe/rolling.py b/dask/dataframe/rolling.py
--- a/dask/dataframe/rolling.py
+++ b/dask/dataframe/rolling.py
@@ -86,7 +86,7 @@ def map_overlap(func, df, before, after, *args, **kwargs):
         meta = kwargs.pop('meta')
     else:
         meta = _emulate(func, df, *args, **kwargs)
-    meta = make_meta(meta)
+    meta = make_meta(meta, index=df._meta.index)
 
     name = '{0}-{1}'.format(func_name, token)
     name_a = 'overlap-prepend-' + tokenize(df, before)
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -264,13 +264,13 @@ def _empty_series(name, dtype, index=None):
 
 
 @make_meta.register((pd.Series, pd.DataFrame))
-def make_meta_pandas(x):
+def make_meta_pandas(x, index=None):
     return x.iloc[:0]
 
 
 @make_meta.register(pd.Index)
-def make_meta_index(x):
-    return x[:0]
+def make_meta_index(x, index=None):
+    return x[0:0]
 
 
 @make_meta.register(object)
@@ -305,7 +305,8 @@ def make_meta_object(x, index=None):
     elif is_arraylike(x):
         return x[:0]
 
-    index = index if index is None else index[0:0]
+    if index is not None:
+        index = make_meta(index)
 
     if isinstance(x, dict):
         return pd.DataFrame({c: _empty_series(c, d, index=index)


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -615,6 +615,22 @@ def test_map_partitions_method_names():
     assert b.dtype == 'i8'
 
 
+def test_map_partitions_propagates_index_metadata():
+    index = pd.Series(list('abcde'), name='myindex')
+    df = pd.DataFrame({'A': np.arange(5, dtype=np.int32),
+                       'B': np.arange(10, 15, dtype=np.int32)},
+                      index=index)
+    ddf = dd.from_pandas(df, npartitions=2)
+    res = ddf.map_partitions(lambda df: df.assign(C=df.A + df.B),
+                             meta=[('A', 'i4'), ('B', 'i4'), ('C', 'i4')])
+    sol = df.assign(C=df.A + df.B)
+    assert_eq(res, sol)
+
+    res = ddf.map_partitions(lambda df: df.rename_axis("newindex"))
+    sol = df.rename_axis("newindex")
+    assert_eq(res, sol)
+
+
 @pytest.mark.xfail(reason='now we use SubgraphCallables')
 def test_map_partitions_keeps_kwargs_readable():
     df = pd.DataFrame({'x': [1, 2, 3, 4], 'y': [5, 6, 7, 8]})
@@ -946,16 +962,21 @@ def test_assign_callable():
 
 
 def test_map():
-    assert_eq(d.a.map(lambda x: x + 1), full.a.map(lambda x: x + 1))
-    lk = dict((v, v + 1) for v in full.a.values)
-    assert_eq(d.a.map(lk), full.a.map(lk))
-    assert_eq(d.b.map(lk), full.b.map(lk))
+    df = pd.DataFrame({'a': range(9),
+                       'b': [4, 5, 6, 1, 2, 3, 0, 0, 0]},
+                      index=pd.Index([0, 1, 3, 5, 6, 8, 9, 9, 9], name='myindex'))
+    ddf = dd.from_pandas(df, npartitions=3)
+
+    assert_eq(ddf.a.map(lambda x: x + 1), df.a.map(lambda x: x + 1))
+    lk = dict((v, v + 1) for v in df.a.values)
+    assert_eq(ddf.a.map(lk), df.a.map(lk))
+    assert_eq(ddf.b.map(lk), df.b.map(lk))
     lk = pd.Series(lk)
-    assert_eq(d.a.map(lk), full.a.map(lk))
-    assert_eq(d.b.map(lk), full.b.map(lk))
-    assert_eq(d.b.map(lk, meta=d.b), full.b.map(lk))
-    assert_eq(d.b.map(lk, meta=('b', 'i8')), full.b.map(lk))
-    pytest.raises(TypeError, lambda: d.a.map(d.b))
+    assert_eq(ddf.a.map(lk), df.a.map(lk))
+    assert_eq(ddf.b.map(lk), df.b.map(lk))
+    assert_eq(ddf.b.map(lk, meta=ddf.b), df.b.map(lk))
+    assert_eq(ddf.b.map(lk, meta=('b', 'i8')), df.b.map(lk))
+    pytest.raises(TypeError, lambda: ddf.a.map(d.b))
 
 
 def test_concat():
diff --git a/dask/dataframe/tests/test_rolling.py b/dask/dataframe/tests/test_rolling.py
--- a/dask/dataframe/tests/test_rolling.py
+++ b/dask/dataframe/tests/test_rolling.py
@@ -52,7 +52,7 @@ def test_map_overlap(npartitions):
         assert_eq(res, sol)
 
 
-def test_map_partitions_names():
+def test_map_overlap_names():
     npartitions = 3
     ddf = dd.from_pandas(df, npartitions)
 
@@ -70,7 +70,7 @@ def test_map_partitions_names():
     assert res4._name != res._name
 
 
-def test_map_partitions_errors():
+def test_map_overlap_errors():
     # Non-integer
     with pytest.raises(ValueError):
         ddf.map_overlap(shifted_sum, 0.5, 3, 0, 2, c=2)
@@ -89,6 +89,18 @@ def test_map_partitions_errors():
                         0, 2, c=2)
 
 
+def test_map_overlap_provide_meta():
+    df = pd.DataFrame({'x': [1, 2, 4, 7, 11],
+                       'y': [1., 2., 3., 4., 5.]}).rename_axis('myindex')
+    ddf = dd.from_pandas(df, npartitions=2)
+
+    # Provide meta spec, but not full metadata
+    res = ddf.map_overlap(lambda df: df.rolling(2).sum(), 2, 0,
+                          meta={'x': 'i8', 'y': 'i8'})
+    sol = df.rolling(2).sum()
+    assert_eq(res, sol)
+
+
 def mad(x):
     return np.fabs(x - x.mean()).mean()