Repository: dask/dask

Issue:
Add documentation for Rolling
This should perhaps both be in the Rolling object as added as API docs.  Doing this well probably requires that we drop 0.17 Pandas support

https://github.com/dask/dask/pull/1221
#1240



Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -431,6 +431,97 @@ def map_partitions(self, func, *args, **kwargs):
         """
         return map_partitions(func, self, *args, **kwargs)
 
+    @insert_meta_param_description(pad=12)
+    def map_overlap(self, func, before, after, *args, **kwargs):
+        """Apply a function to each partition, sharing rows with adjacent partitions.
+
+        This can be useful for implementing windowing functions such as
+        ``df.rolling(...).mean()`` or ``df.diff()``.
+
+        Parameters
+        ----------
+        func : function
+            Function applied to each partition.
+        before : int
+            The number of rows to prepend to partition ``i`` from the end of
+            partition ``i - 1``.
+        after : int
+            The number of rows to append to partition ``i`` from the beginning
+            of partition ``i + 1``.
+        args, kwargs :
+            Arguments and keywords to pass to the function. The partition will
+            be the first argument, and these will be passed *after*.
+        $META
+
+        Notes
+        -----
+        Given positive integers ``before`` and ``after``, and a function
+        ``func``, ``map_overlap`` does the following:
+
+        1. Prepend ``before`` rows to each partition ``i`` from the end of
+           partition ``i - 1``. The first partition has no rows prepended.
+
+        2. Append ``after`` rows to each partition ``i`` from the beginning of
+           partition ``i + 1``. The last partition has no rows appended.
+
+        3. Apply ``func`` to each partition, passing in any extra ``args`` and
+           ``kwargs`` if provided.
+
+        4. Trim ``before`` rows from the beginning of all but the first
+           partition.
+
+        5. Trim ``after`` rows from the end of all but the last partition.
+
+        Note that the index and divisions are assumed to remain unchanged.
+
+        Examples
+        --------
+        Given a DataFrame, Series, or Index, such as:
+
+        >>> import dask.dataframe as dd
+        >>> df = pd.DataFrame({'x': [1, 2, 4, 7, 11],
+        ...                    'y': [1., 2., 3., 4., 5.]})
+        >>> ddf = dd.from_pandas(df, npartitions=2)
+
+        A rolling sum with a trailing moving window of size 2 can be computed by
+        overlapping 2 rows before each partition, and then mapping calls to
+        ``df.rolling(2).sum()``:
+
+        >>> ddf.compute()
+            x    y
+        0   1  1.0
+        1   2  2.0
+        2   4  3.0
+        3   7  4.0
+        4  11  5.0
+        >>> ddf.map_overlap(lambda df: df.rolling(2).sum(), 2, 0).compute()
+              x    y
+        0   NaN  NaN
+        1   3.0  3.0
+        2   6.0  5.0
+        3  11.0  7.0
+        4  18.0  9.0
+
+        The pandas ``diff`` method computes a discrete difference shifted by a
+        number of periods (can be positive or negative). This can be
+        implemented by mapping calls to ``df.diff`` to each partition after
+        prepending/appending that many rows, depending on sign:
+
+        >>> def diff(df, periods=1):
+        ...     before, after = (periods, 0) if periods > 0 else (0, -periods)
+        ...     return df.map_overlap(lambda df, periods=1: df.diff(periods),
+        ...                           periods, 0, periods=periods)
+        >>> diff(ddf, 1).compute()
+             x    y
+        0  NaN  NaN
+        1  1.0  1.0
+        2  2.0  1.0
+        3  3.0  1.0
+        4  4.0  1.0
+        """
+        from .rolling import map_overlap
+        return map_overlap(func, self, before, after, *args, **kwargs)
+
     @insert_meta_param_description(pad=12)
     def reduction(self, chunk, aggregate=None, combine=None, meta=no_default,
                   token=None, split_every=None, chunk_kwargs=None,
@@ -449,6 +540,7 @@ def reduction(self, chunk, aggregate=None, combine=None, meta=no_default,
 
             The input to ``aggregate`` depends on the output of ``chunk``.
             If the output of ``chunk`` is a:
+
             - scalar: Input is a Series, with one row per partition.
             - Series: Input is a DataFrame, with one row per partition. Columns
               are the rows in the output series.
@@ -1047,6 +1139,20 @@ def rolling(self, window, min_periods=None, freq=None, center=False,
         return Rolling(self, window=window, min_periods=min_periods,
                        freq=freq, center=center, win_type=win_type, axis=axis)
 
+    @derived_from(pd.DataFrame)
+    def diff(self, periods=1, axis=0):
+        axis = self._validate_axis(axis)
+        if not isinstance(periods, int):
+            raise TypeError("periods must be an integer")
+
+        if axis == 1:
+            return self.map_partitions(M.diff, token='diff', periods=periods,
+                                       axis=1)
+
+        before, after = (periods, 0) if periods > 0 else (0, -periods)
+        return self.map_overlap(M.diff, before, after, token='diff',
+                                periods=periods)
+
     def _reduction_agg(self, name, axis=None, skipna=True,
                        split_every=False):
         axis = self._validate_axis(axis)
diff --git a/dask/dataframe/rolling.py b/dask/dataframe/rolling.py
--- a/dask/dataframe/rolling.py
+++ b/dask/dataframe/rolling.py
@@ -1,94 +1,149 @@
 from __future__ import absolute_import, division, print_function
 
-from functools import partial, wraps
+import warnings
+from functools import wraps
 
-from toolz import merge
 import pandas as pd
+from pandas.core.window import Rolling as pd_Rolling
 
 from ..base import tokenize
-from ..utils import M
+from ..utils import M, funcname, derived_from
+from .core import _emulate
+from .utils import make_meta
+
+
+def overlap_chunk(func, prev_part, current_part, next_part, before, after,
+                  args, kwargs):
+    if ((prev_part is not None and prev_part.shape[0] != before) or
+            (next_part is not None and next_part.shape[0] != after)):
+        raise NotImplementedError("Partition size is less than overlapping "
+                                  "window size. Try using ``df.repartition`` "
+                                  "to increase the partition size.")
+    parts = [p for p in (prev_part, current_part, next_part) if p is not None]
+    combined = pd.concat(parts)
+    out = func(combined, *args, **kwargs)
+    if prev_part is None:
+        before = None
+    if next_part is None:
+        return out.iloc[before:]
+    return out.iloc[before:-after]
+
+
+def map_overlap(func, df, before, after, *args, **kwargs):
+    """Apply a function to each partition, sharing rows with adjacent partitions.
+
+    Parameters
+    ----------
+    func : function
+        Function applied to each partition.
+    df : dd.DataFrame, dd.Series
+    before : int
+        The number of rows to prepend to partition ``i`` from the end of
+        partition ``i - 1``.
+    after : int
+        The number of rows to append to partition ``i`` from the beginning
+        of partition ``i + 1``.
+    args, kwargs :
+        Arguments and keywords to pass to the function. The partition will
+        be the first argument, and these will be passed *after*.
+
+    See Also
+    --------
+    dd.DataFrame.map_overlap
+    """
+    if not (isinstance(before, int) and before >= 0 and
+            isinstance(after, int) and after >= 0):
+        raise ValueError("before and after must be positive integers")
 
+    if 'token' in kwargs:
+        func_name = kwargs.pop('token')
+        token = tokenize(df, before, after, *args, **kwargs)
+    else:
+        func_name = 'overlap-' + funcname(func)
+        token = tokenize(func, df, before, after, *args, **kwargs)
 
-def rolling_chunk(func, part1, part2, window, *args):
-    if part1.shape[0] < window - 1:
-        raise NotImplementedError("Window larger than partition size")
-    if window > 1:
-        extra = window - 1
-        combined = pd.concat([part1.iloc[-extra:], part2])
-        applied = func(combined, window, *args)
-        return applied.iloc[extra:]
+    if 'meta' in kwargs:
+        meta = kwargs.pop('meta')
     else:
-        return func(part2, window, *args)
+        meta = _emulate(func, df, *args, **kwargs)
+    meta = make_meta(meta)
+
+    name = '{0}-{1}'.format(func_name, token)
+    name_a = 'overlap-prepend-' + tokenize(df, before)
+    name_b = 'overlap-append-' + tokenize(df, after)
+    df_name = df._name
+
+    dsk = df.dask.copy()
+    if before:
+        dsk.update({(name_a, i): (M.tail, (df_name, i), before)
+                    for i in range(df.npartitions - 1)})
+        prevs = [None] + [(name_a, i) for i in range(df.npartitions - 1)]
+    else:
+        prevs = [None] * df.npartitions
+
+    if after:
+        dsk.update({(name_b, i): (M.head, (df_name, i), after)
+                    for i in range(1, df.npartitions)})
+        nexts = [(name_b, i) for i in range(1, df.npartitions)] + [None]
+    else:
+        nexts = [None] * df.npartitions
+
+    for i, (prev, current, next) in enumerate(zip(prevs, df._keys(), nexts)):
+        dsk[(name, i)] = (overlap_chunk, func, prev, current, next, before,
+                          after, args, kwargs)
 
+    return df._constructor(dsk, name, meta, df.divisions)
 
-def wrap_rolling(func):
+
+def wrap_rolling(func, method_name):
     """Create a chunked version of a pandas.rolling_* function"""
     @wraps(func)
     def rolling(arg, window, *args, **kwargs):
-        if not isinstance(window, int):
-            raise TypeError('Window must be an integer')
-        if window < 0:
-            raise ValueError('Window must be a positive integer')
-        if 'freq' in kwargs or 'how' in kwargs:
-            raise NotImplementedError('Resampling before rolling computations '
-                                      'not supported')
-        old_name = arg._name
-        token = tokenize(func, arg, window, args, kwargs)
-        new_name = 'rolling-' + token
-        f = partial(func, **kwargs)
-        dsk = {(new_name, 0): (f, (old_name, 0), window) + args}
-        for i in range(1, arg.npartitions + 1):
-            dsk[(new_name, i)] = (rolling_chunk, f, (old_name, i - 1),
-                                  (old_name, i), window) + args
-        return arg._constructor(merge(arg.dask, dsk), new_name,
-                                arg, arg.divisions)
+        # pd.rolling_* functions are deprecated
+        warnings.warn(("DeprecationWarning: dd.rolling_{0} is deprecated and "
+                       "will be removed in a future version, replace with "
+                       "df.rolling(...).{0}(...)").format(method_name))
+
+        rolling_kwargs = {}
+        method_kwargs = {}
+        for k, v in kwargs.items():
+            if k in {'min_periods', 'center', 'win_type', 'axis', 'freq'}:
+                rolling_kwargs[k] = v
+            else:
+                method_kwargs[k] = v
+        rolling = arg.rolling(window, **rolling_kwargs)
+        return getattr(rolling, method_name)(*args, **method_kwargs)
     return rolling
 
 
-rolling_count = wrap_rolling(pd.rolling_count)
-rolling_sum = wrap_rolling(pd.rolling_sum)
-rolling_mean = wrap_rolling(pd.rolling_mean)
-rolling_median = wrap_rolling(pd.rolling_median)
-rolling_min = wrap_rolling(pd.rolling_min)
-rolling_max = wrap_rolling(pd.rolling_max)
-rolling_std = wrap_rolling(pd.rolling_std)
-rolling_var = wrap_rolling(pd.rolling_var)
-rolling_skew = wrap_rolling(pd.rolling_skew)
-rolling_kurt = wrap_rolling(pd.rolling_kurt)
-rolling_quantile = wrap_rolling(pd.rolling_quantile)
-rolling_apply = wrap_rolling(pd.rolling_apply)
-rolling_window = wrap_rolling(pd.rolling_window)
+rolling_count = wrap_rolling(pd.rolling_count, 'count')
+rolling_sum = wrap_rolling(pd.rolling_sum, 'sum')
+rolling_mean = wrap_rolling(pd.rolling_mean, 'mean')
+rolling_median = wrap_rolling(pd.rolling_median, 'median')
+rolling_min = wrap_rolling(pd.rolling_min, 'min')
+rolling_max = wrap_rolling(pd.rolling_max, 'max')
+rolling_std = wrap_rolling(pd.rolling_std, 'std')
+rolling_var = wrap_rolling(pd.rolling_var, 'var')
+rolling_skew = wrap_rolling(pd.rolling_skew, 'skew')
+rolling_kurt = wrap_rolling(pd.rolling_kurt, 'kurt')
+rolling_quantile = wrap_rolling(pd.rolling_quantile, 'quantile')
+rolling_apply = wrap_rolling(pd.rolling_apply, 'apply')
 
 
-def call_pandas_rolling_method_single(this_partition, rolling_kwargs,
-                                      method_name, method_args, method_kwargs):
-    # used for the start of the df/series (or for rolling through columns)
-    method = getattr(this_partition.rolling(**rolling_kwargs), method_name)
-    return method(*method_args, **method_kwargs)
+@wraps(pd.rolling_window)
+def rolling_window(arg, window, **kwargs):
+    if kwargs.pop('mean', True):
+        return rolling_mean(arg, window, **kwargs)
+    return rolling_sum(arg, window, **kwargs)
 
 
-def call_pandas_rolling_method_with_neighbors(prev_partition, this_partition,
-                                              next_partition, before, after,
-                                              rolling_kwargs, method_name,
-                                              method_args, method_kwargs):
-
-    if prev_partition.shape[0] != before or next_partition.shape[0] != after:
-        raise NotImplementedError("Window requires larger inter-partition view than partition size")
-
-    combined = pd.concat([prev_partition, this_partition, next_partition])
-    method = getattr(combined.rolling(**rolling_kwargs), method_name)
-    applied = method(*method_args, **method_kwargs)
-    if after:
-        return applied.iloc[before:-after]
-    else:
-        return applied.iloc[before:]
+def pandas_rolling_method(df, rolling_kwargs, name, *args, **kwargs):
+    rolling = df.rolling(**rolling_kwargs)
+    return getattr(rolling, name)(*args, **kwargs)
 
 
 class Rolling(object):
-    # What you get when you do ddf.rolling(...) or similar
-    """Provides rolling window calculations.
-
-    """
+    """Provides rolling window calculations."""
 
     def __init__(self, obj, window=None, min_periods=None, freq=None,
                  center=False, win_type=None, axis=0):
@@ -97,138 +152,92 @@ def __init__(self, obj, window=None, min_periods=None, freq=None,
             raise NotImplementedError(msg)
 
         self.obj = obj     # dataframe or series
-
         self.window = window
         self.min_periods = min_periods
         self.center = center
         self.win_type = win_type
         self.axis = axis
-
         # Allow pandas to raise if appropriate
         obj._meta.rolling(**self._rolling_kwargs())
 
     def _rolling_kwargs(self):
-        return {
-            'window': self.window,
-            'min_periods': self.min_periods,
-            'center': self.center,
-            'win_type': self.win_type,
-            'axis': self.axis}
+        return {'window': self.window,
+                'min_periods': self.min_periods,
+                'center': self.center,
+                'win_type': self.win_type,
+                'axis': self.axis}
 
     def _call_method(self, method_name, *args, **kwargs):
-        # make sure dask does not mistake this for a task
-        args = list(args)
-
-        old_name = self.obj._name
-        new_name = 'rolling-' + tokenize(
-            self.obj, self._rolling_kwargs(), method_name, args, kwargs)
-
-        dsk = {}
-        if self.axis in [1, 'columns'] or self.window <= 1 or self.obj.npartitions == 1:
-            # This is the easy scenario, we're rolling over columns (or not
-            # really rolling at all, so each chunk is independent.
-            for i in range(self.obj.npartitions):
-                dsk[new_name, i] = (
-                    call_pandas_rolling_method_single, (old_name, i),
-                    self._rolling_kwargs(), method_name, args, kwargs)
+        rolling_kwargs = self._rolling_kwargs()
+        meta = pandas_rolling_method(self.obj._meta_nonempty, rolling_kwargs,
+                                     method_name, *args, **kwargs)
+
+        if (self.axis in (1, 'columns') or self.window <= 1 or
+                self.obj.npartitions == 1):
+            # There's no overlap just use map_partitions
+            return self.obj.map_partitions(pandas_rolling_method,
+                                           rolling_kwargs, method_name,
+                                           *args, token=method_name, meta=meta,
+                                           **kwargs)
+        # Convert window to overlap
+        if self.center:
+            before = self.window // 2
+            after = self.window - before - 1
         else:
-            # This is a bit trickier, we need to feed in information from the
-            # neighbors to roll along rows.
-
-            # Figure out how many we need to look at before and after.
-            if self.center:
-                before = self.window // 2
-                after = self.window - before - 1
-            else:
-                before = self.window - 1
-                after = 0
-
-            head_name = 'head-{}-{}'.format(after, old_name)
-            tail_name = 'tail-{}-{}'.format(before, old_name)
-
-            # First chunk, only look after (if necessary)
-            if after > 0:
-                next_partition = (head_name, 1)
-                dsk[next_partition] = (M.head, (old_name, 1), after)
-            else:
-                # Either we are only looking backward or this was the
-                # only chunk.
-                next_partition = self.obj._meta
-            dsk[new_name, 0] = (call_pandas_rolling_method_with_neighbors,
-                                self.obj._meta, (old_name, 0),
-                                next_partition, 0, after,
-                                self._rolling_kwargs(), method_name,
-                                args, kwargs)
-
-            # All the middle chunks
-            for i in range(1, self.obj.npartitions - 1):
-                # Get just the needed values from the previous partition
-                dsk[tail_name, i - 1] = (M.tail, (old_name, i - 1), before)
-
-                if after:
-                    next_partition = (head_name, i + 1)
-                    dsk[next_partition] = (M.head, (old_name, i + 1), after)
-
-                dsk[new_name, i] = (call_pandas_rolling_method_with_neighbors,
-                                    (tail_name, i - 1), (old_name, i),
-                                    next_partition, before, after,
-                                    self._rolling_kwargs(), method_name,
-                                    args, kwargs)
-
-            # The last chunk
-            if self.obj.npartitions > 1:
-                # if the first wasn't the only partition
-                end = self.obj.npartitions - 1
-                dsk[tail_name, end - 1] = (M.tail, (old_name, end - 1), before)
-
-                dsk[new_name, end] = (
-                    call_pandas_rolling_method_with_neighbors,
-                    (tail_name, end - 1), (old_name, end), self.obj._meta,
-                    before, 0, self._rolling_kwargs(), method_name, args,
-                    kwargs)
-
-        # Do the pandas operation to get the appropriate thing for metadata
-        pd_rolling = self.obj._meta.rolling(**self._rolling_kwargs())
-        metadata = getattr(pd_rolling, method_name)(*args, **kwargs)
+            before = self.window - 1
+            after = 0
 
-        return self.obj._constructor(merge(self.obj.dask, dsk),
-                                     new_name, metadata, self.obj.divisions)
+        return map_overlap(pandas_rolling_method, self.obj, before, after,
+                           rolling_kwargs, method_name, *args,
+                           token=method_name, meta=meta, **kwargs)
 
-    def count(self, *args, **kwargs):
-        return self._call_method('count', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def count(self):
+        return self._call_method('count')
 
-    def sum(self, *args, **kwargs):
-        return self._call_method('sum', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def sum(self):
+        return self._call_method('sum')
 
-    def mean(self, *args, **kwargs):
-        return self._call_method('mean', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def mean(self):
+        return self._call_method('mean')
 
-    def median(self, *args, **kwargs):
-        return self._call_method('median', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def median(self):
+        return self._call_method('median')
 
-    def min(self, *args, **kwargs):
-        return self._call_method('min', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def min(self):
+        return self._call_method('min')
 
-    def max(self, *args, **kwargs):
-        return self._call_method('max', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def max(self):
+        return self._call_method('max')
 
-    def std(self, *args, **kwargs):
-        return self._call_method('std', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def std(self, ddof=1):
+        return self._call_method('std', ddof=1)
 
-    def var(self, *args, **kwargs):
-        return self._call_method('var', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def var(self, ddof=1):
+        return self._call_method('var', ddof=1)
 
-    def skew(self, *args, **kwargs):
-        return self._call_method('skew', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def skew(self):
+        return self._call_method('skew')
 
-    def kurt(self, *args, **kwargs):
-        return self._call_method('kurt', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def kurt(self):
+        return self._call_method('kurt')
 
-    def quantile(self, *args, **kwargs):
-        return self._call_method('quantile', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def quantile(self, quantile):
+        return self._call_method('quantile', quantile)
 
-    def apply(self, *args, **kwargs):
-        return self._call_method('apply', *args, **kwargs)
+    @derived_from(pd_Rolling)
+    def apply(self, func, args=(), kwargs={}):
+        return self._call_method('apply', func, args=args, kwargs=kwargs)
 
     def __repr__(self):
         return 'Rolling [{}]'.format(','.join(
diff --git a/dask/dataframe/tseries/resample.py b/dask/dataframe/tseries/resample.py
--- a/dask/dataframe/tseries/resample.py
+++ b/dask/dataframe/tseries/resample.py
@@ -4,9 +4,11 @@
 
 import pandas as pd
 import numpy as np
+from pandas.tseries.resample import Resampler as pd_Resampler
 
 from ..core import DataFrame, Series
 from ...base import tokenize
+from ...utils import derived_from
 
 
 def getnanos(rule):
@@ -121,41 +123,54 @@ def _agg(self, how, meta=None, fill_value=np.nan):
             return DataFrame(dsk, name, meta, outdivs)
         return Series(dsk, name, meta, outdivs)
 
+    @derived_from(pd_Resampler)
     def count(self):
         return self._agg('count', fill_value=0)
 
+    @derived_from(pd_Resampler)
     def first(self):
         return self._agg('first')
 
+    @derived_from(pd_Resampler)
     def last(self):
         return self._agg('last')
 
+    @derived_from(pd_Resampler)
     def mean(self):
         return self._agg('mean')
 
+    @derived_from(pd_Resampler)
     def min(self):
         return self._agg('min')
 
+    @derived_from(pd_Resampler)
     def median(self):
         return self._agg('median')
 
+    @derived_from(pd_Resampler)
     def max(self):
         return self._agg('max')
 
+    @derived_from(pd_Resampler)
     def ohlc(self):
         return self._agg('ohlc')
 
+    @derived_from(pd_Resampler)
     def prod(self):
         return self._agg('prod')
 
+    @derived_from(pd_Resampler)
     def sem(self):
         return self._agg('sem')
 
+    @derived_from(pd_Resampler)
     def std(self):
         return self._agg('std')
 
+    @derived_from(pd_Resampler)
     def sum(self):
         return self._agg('sum')
 
+    @derived_from(pd_Resampler)
     def var(self):
         return self._agg('var')


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -2548,3 +2548,24 @@ def test_set_index_sorted_min_max_same():
 
     df2 = df.set_index('y', sorted=True)
     assert df2.divisions == (0, 1, 1)
+
+
+def test_diff():
+    df = pd.DataFrame(np.random.randn(100, 5), columns=list('abcde'))
+    ddf = dd.from_pandas(df, 5)
+
+    assert_eq(ddf.diff(), df.diff())
+    assert_eq(ddf.diff(0), df.diff(0))
+    assert_eq(ddf.diff(2), df.diff(2))
+    assert_eq(ddf.diff(-2), df.diff(-2))
+
+    assert_eq(ddf.diff(2, axis=1), df.diff(2, axis=1))
+
+    assert_eq(ddf.a.diff(), df.a.diff())
+    assert_eq(ddf.a.diff(0), df.a.diff(0))
+    assert_eq(ddf.a.diff(2), df.a.diff(2))
+    assert_eq(ddf.a.diff(-2), df.a.diff(-2))
+
+    assert ddf.diff(2)._name == ddf.diff(2)._name
+    assert ddf.diff(2)._name != ddf.diff(3)._name
+    pytest.raises(TypeError, lambda: ddf.diff(1.5))
diff --git a/dask/dataframe/tests/test_rolling.py b/dask/dataframe/tests/test_rolling.py
--- a/dask/dataframe/tests/test_rolling.py
+++ b/dask/dataframe/tests/test_rolling.py
@@ -4,7 +4,67 @@
 
 import dask.dataframe as dd
 from dask.dataframe.utils import assert_eq
-from dask.utils import ignoring
+
+N = 40
+df = pd.DataFrame({'a': np.random.randn(N).cumsum(),
+                   'b': np.random.randint(100, size=(N,)),
+                   'c': np.random.randint(100, size=(N,)),
+                   'd': np.random.randint(100, size=(N,)),
+                   'e': np.random.randint(100, size=(N,))})
+ddf = dd.from_pandas(df, 3)
+
+
+def shifted_sum(df, before, after, c=0):
+    a = df.shift(before)
+    b = df.shift(-after)
+    return df + a + b + c
+
+
+@pytest.mark.parametrize('npartitions', [1, 4])
+def test_map_overlap(npartitions):
+    ddf = dd.from_pandas(df, npartitions)
+    for before, after in [(0, 3), (3, 0), (3, 3), (0, 0)]:
+        # DataFrame
+        res = ddf.map_overlap(shifted_sum, before, after, before, after, c=2)
+        sol = shifted_sum(df, before, after, c=2)
+        assert_eq(res, sol)
+
+        # Series
+        res = ddf.b.map_overlap(shifted_sum, before, after, before, after, c=2)
+        sol = shifted_sum(df.b, before, after, c=2)
+        assert_eq(res, sol)
+
+
+def test_map_partitions_names():
+    npartitions = 3
+    ddf = dd.from_pandas(df, npartitions)
+
+    res = ddf.map_overlap(shifted_sum, 0, 3, 0, 3, c=2)
+    res2 = ddf.map_overlap(shifted_sum, 0, 3, 0, 3, c=2)
+    assert set(res.dask) == set(res2.dask)
+
+    res3 = ddf.map_overlap(shifted_sum, 0, 3, 0, 3, c=3)
+    assert res3._name != res._name
+    # Difference is just the final map
+    diff = set(res3.dask).difference(res.dask)
+    assert len(diff) == npartitions
+
+    res4 = ddf.map_overlap(shifted_sum, 3, 0, 0, 3, c=2)
+    assert res4._name != res._name
+
+
+def test_map_partitions_errors():
+    # Non-integer
+    with pytest.raises(ValueError):
+        ddf.map_overlap(shifted_sum, 0.5, 3, 0, 2, c=2)
+
+    # Negative
+    with pytest.raises(ValueError):
+        ddf.map_overlap(shifted_sum, 0, -5, 0, 2, c=2)
+
+    # Partition size < window size
+    with pytest.raises(NotImplementedError):
+        ddf.map_overlap(shifted_sum, 0, 100, 0, 100, c=2).compute()
 
 
 def mad(x):
@@ -28,9 +88,8 @@ def rolling_functions_tests(p, d):
               dd.rolling_kurt(d, 3), check_less_precise=True)
     assert_eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
     assert_eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
-    with ignoring(ImportError):
-        assert_eq(pd.rolling_window(p, 3, 'boxcar'),
-                  dd.rolling_window(d, 3, 'boxcar'))
+    assert_eq(pd.rolling_window(p, 3, win_type='boxcar'),
+              dd.rolling_window(d, 3, win_type='boxcar'))
     # Test with edge-case window sizes
     assert_eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
     assert_eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
@@ -39,60 +98,12 @@ def rolling_functions_tests(p, d):
               dd.rolling_sum(d, 3, min_periods=3))
 
 
-def basic_rolling_tests(p, d):
-    # Works for series or df
-
-    # New rolling API
-    assert_eq(p.rolling(3).count(), d.rolling(3).count())
-    assert_eq(p.rolling(3).sum(), d.rolling(3).sum())
-    assert_eq(p.rolling(3).mean(), d.rolling(3).mean())
-    assert_eq(p.rolling(3).median(), d.rolling(3).median())
-    assert_eq(p.rolling(3).min(), d.rolling(3).min())
-    assert_eq(p.rolling(3).max(), d.rolling(3).max())
-    assert_eq(p.rolling(3).std(), d.rolling(3).std())
-    assert_eq(p.rolling(3).var(), d.rolling(3).var())
-    # see note around test_rolling_dataframe for logic concerning precision
-    assert_eq(p.rolling(3).skew(),
-              d.rolling(3).skew(), check_less_precise=True)
-    assert_eq(p.rolling(3).kurt(),
-              d.rolling(3).kurt(), check_less_precise=True)
-    assert_eq(p.rolling(3).quantile(0.5), d.rolling(3).quantile(0.5))
-    assert_eq(p.rolling(3).apply(mad), d.rolling(3).apply(mad))
-    with ignoring(ImportError):
-        assert_eq(p.rolling(3, win_type='boxcar').sum(),
-                  d.rolling(3, win_type='boxcar').sum())
-    # Test with edge-case window sizes
-    assert_eq(p.rolling(0).sum(), d.rolling(0).sum())
-    assert_eq(p.rolling(1).sum(), d.rolling(1).sum())
-    # Test with kwargs
-    assert_eq(p.rolling(3, min_periods=2).sum(),
-              d.rolling(3, min_periods=2).sum())
-    # Test with center
-    assert_eq(p.rolling(3, center=True).max(),
-              d.rolling(3, center=True).max())
-    assert_eq(p.rolling(3, center=False).std(),
-              d.rolling(3, center=False).std())
-    assert_eq(p.rolling(6, center=True).var(),
-              d.rolling(6, center=True).var())
-    # see note around test_rolling_dataframe for logic concerning precision
-    assert_eq(p.rolling(7, center=True).skew(),
-              d.rolling(7, center=True).skew(),
-              check_less_precise=True)
-
-
 def test_rolling_functions_series():
     ts = pd.Series(np.random.randn(25).cumsum())
     dts = dd.from_pandas(ts, 3)
     rolling_functions_tests(ts, dts)
 
 
-def test_rolling_series():
-    for ts in [pd.Series(np.random.randn(25).cumsum()),
-               pd.Series(np.random.randint(100, size=(25,)))]:
-        dts = dd.from_pandas(ts, 3)
-        basic_rolling_tests(ts, dts)
-
-
 def test_rolling_functions_dataframe():
     df = pd.DataFrame({'a': np.random.randn(25).cumsum(),
                        'b': np.random.randint(100, size=(25,))})
@@ -100,7 +111,6 @@ def test_rolling_functions_dataframe():
     rolling_functions_tests(df, ddf)
 
 
-@pytest.mark.parametrize('npartitions', [1, 2, 3])
 @pytest.mark.parametrize('method,args,check_less_precise', [
     ('count', (), False),
     ('sum', (), False),
@@ -118,39 +128,24 @@ def test_rolling_functions_dataframe():
                           # So far, I am only weakening the check for kurt and skew,
                           # as they involve third degree powers and higher
     ('quantile', (.38,), False),
-    ('apply', (np.sum,), False),
+    ('apply', (mad,), False),
 ])
 @pytest.mark.parametrize('window', [1, 2, 4, 5])
 @pytest.mark.parametrize('center', [True, False])
-@pytest.mark.parametrize('axis', [0, 'columns'])
-def test_rolling_dataframe(npartitions, method, args, window, center, axis,
-                           check_less_precise):
-    if method == 'count' and axis in [1, 'columns']:
-        pytest.xfail('count currently ignores the axis argument.')
-
-    N = 40
-    df = pd.DataFrame({'a': np.random.randn(N).cumsum(),
-                       'b': np.random.randint(100, size=(N,)),
-                       'c': np.random.randint(100, size=(N,)),
-                       'd': np.random.randint(100, size=(N,)),
-                       'e': np.random.randint(100, size=(N,))})
-    ddf = dd.from_pandas(df, npartitions)
-
-    prolling = df.rolling(window, center=center, axis=axis)
-    drolling = ddf.rolling(window, center=center, axis=axis)
+def test_rolling_methods(method, args, window, center, check_less_precise):
+    # DataFrame
+    prolling = df.rolling(window, center=center)
+    drolling = ddf.rolling(window, center=center)
     assert_eq(getattr(prolling, method)(*args),
               getattr(drolling, method)(*args),
               check_less_precise=check_less_precise)
 
-
-def test_rolling_functions_raises():
-    df = pd.DataFrame({'a': np.random.randn(25).cumsum(),
-                       'b': np.random.randint(100, size=(25,))})
-    ddf = dd.from_pandas(df, 3)
-    pytest.raises(TypeError, lambda: dd.rolling_mean(ddf, 1.5))
-    pytest.raises(ValueError, lambda: dd.rolling_mean(ddf, -1))
-    pytest.raises(NotImplementedError, lambda: dd.rolling_mean(ddf, 3, freq=2))
-    pytest.raises(NotImplementedError, lambda: dd.rolling_mean(ddf, 3, how='min'))
+    # Series
+    prolling = df.a.rolling(window, center=center)
+    drolling = ddf.a.rolling(window, center=center)
+    assert_eq(getattr(prolling, method)(*args),
+              getattr(drolling, method)(*args),
+              check_less_precise=check_less_precise)
 
 
 def test_rolling_raises():
@@ -166,13 +161,6 @@ def test_rolling_raises():
     pytest.raises(NotImplementedError, lambda: ddf.rolling(100).mean().compute())
 
 
-def test_rolling_functions_names():
-    df = pd.DataFrame({'a': [1, 2, 3],
-                       'b': [4, 5, 6]})
-    a = dd.from_pandas(df, npartitions=2)
-    assert sorted(dd.rolling_sum(a, 2).dask) == sorted(dd.rolling_sum(a, 2).dask)
-
-
 def test_rolling_names():
     df = pd.DataFrame({'a': [1, 2, 3],
                        'b': [4, 5, 6]})
@@ -198,17 +186,6 @@ def test_rolling_axis():
     assert_eq(s.rolling(5, axis=0).std(), ds.rolling(5, axis=0).std())
 
 
-def test_rolling_function_partition_size():
-    df = pd.DataFrame(np.random.randn(50, 2))
-    ddf = dd.from_pandas(df, npartitions=5)
-
-    for obj, dobj in [(df, ddf), (df[0], ddf[0])]:
-        assert_eq(pd.rolling_mean(obj, 10), dd.rolling_mean(dobj, 10))
-        assert_eq(pd.rolling_mean(obj, 11), dd.rolling_mean(dobj, 11))
-        with pytest.raises(NotImplementedError):
-            dd.rolling_mean(dobj, 12).compute()
-
-
 def test_rolling_partition_size():
     df = pd.DataFrame(np.random.randn(50, 2))
     ddf = dd.from_pandas(df, npartitions=5)
@@ -222,9 +199,9 @@ def test_rolling_partition_size():
 
 def test_rolling_repr():
     ddf = dd.from_pandas(pd.DataFrame([10] * 30), npartitions=3)
-    assert repr(ddf.rolling(4)) in ['Rolling [window=4,center=False,axis=0]',
+    assert repr(ddf.rolling(4)) in {'Rolling [window=4,center=False,axis=0]',
                                     'Rolling [window=4,axis=0,center=False]',
                                     'Rolling [center=False,axis=0,window=4]',
                                     'Rolling [center=False,window=4,axis=0]',
                                     'Rolling [axis=0,window=4,center=False]',
-                                    'Rolling [axis=0,center=False,window=4]']
+                                    'Rolling [axis=0,center=False,window=4]'}