Repository: dask/dask

Issue:
Rolling operations on dask.dataframe
This probably requires a simpler version of `da.ghost`

http://pandas.pydata.org/pandas-docs/stable/computation.html#moving-rolling-statistics-moments



Discussion:
The relevant Pandas API is `pandas.rolling_*`

``` python
In [1]: import pandas as pd

In [2]: s = pd.Series([1, 2, 3, 2, 1, 1, 5, 1])

In [3]: pd.rolling_max(s, 2)
Out[3]: 
0   NaN
1     2
2     3
3     3
4     2
5     1
6     5
7     5
dtype: float64
```

The solution is probably to make a new `dd.Series` with the same `name` and `divisions` where tasks for each partition depend on both the matching partition in the input and in the previous partition

```
dsk[(s.name, i)] = (some_function, ..., (s.name, i), (s.name, i - 1))
```

There are corner cases that this doesn't hit, such as when the size of the rolling window is larger than the size of the previous partition.  I would probably just ignore or raise an error in these cases.

The `some_function` function above probably does something like the following
1.  Take the window-size of entries from the tail of the previous partition
2.  Creates a new pd.Series that concatenates this tail to the beginning of the current partition
3.  Calls the matching pandas rolling_ function on the result
4.  Trims off the few bogus entries at the beginning

There are a few of these functions so it probably makes sense to do this in a general way and then specialize as appropriate.


Resolving patch:
diff --git a/dask/dataframe/__init__.py b/dask/dataframe/__init__.py
--- a/dask/dataframe/__init__.py
+++ b/dask/dataframe/__init__.py
@@ -4,3 +4,7 @@
                  from_bcolz, from_pandas, from_dask_array, read_hdf)
 from .optimize import optimize
 from .multi import merge
+from .rolling import (rolling_count, rolling_sum, rolling_mean, rolling_median,
+                      rolling_min, rolling_max, rolling_std, rolling_var,
+                      rolling_skew, rolling_kurt, rolling_quantile, rolling_apply,
+                      rolling_window)
diff --git a/dask/dataframe/rolling.py b/dask/dataframe/rolling.py
new file mode 100644
--- /dev/null
+++ b/dask/dataframe/rolling.py
@@ -0,0 +1,56 @@
+from functools import partial, wraps
+
+from toolz import merge
+import pandas as pd
+
+from .core import tokens
+
+
+def rolling_chunk(func, part1, part2, window, *args):
+    if part1.shape[0] < window:
+        raise NotImplementedError("Window larger than partition size")
+    if window > 1:
+        extra = window - 1
+        combined = pd.concat([part1.iloc[-extra:], part2])
+        applied = func(combined, window, *args)
+        return applied.iloc[extra:]
+    else:
+        return func(part2, window, *args)
+
+
+def wrap_rolling(func):
+    """Create a chunked version of a pandas.rolling_* function"""
+    @wraps(func)
+    def rolling(arg, window, *args, **kwargs):
+        if not isinstance(window, int):
+            raise TypeError('Window must be an integer')
+        if window < 0:
+            raise ValueError('Window must be a positive integer')
+        if 'freq' in kwargs or 'how' in kwargs:
+            raise NotImplementedError('Resampling before rolling computations '
+                                      'not supported')
+        old_name = arg._name
+        new_name = 'rolling' + next(tokens)
+        f = partial(func, **kwargs)
+        dsk = {(new_name, 0): (f, (old_name, 0), window) + args}
+        for i in range(1, arg.npartitions + 1):
+            dsk[(new_name, i)] = (rolling_chunk, f, (old_name, i - 1),
+                                  (old_name, i), window) + args
+        return type(arg)(merge(arg.dask, dsk), new_name,
+                         arg.column_info, arg.divisions)
+    return rolling
+
+
+rolling_count = wrap_rolling(pd.rolling_count)
+rolling_sum = wrap_rolling(pd.rolling_sum)
+rolling_mean = wrap_rolling(pd.rolling_mean)
+rolling_median = wrap_rolling(pd.rolling_median)
+rolling_min = wrap_rolling(pd.rolling_min)
+rolling_max = wrap_rolling(pd.rolling_max)
+rolling_std = wrap_rolling(pd.rolling_std)
+rolling_var = wrap_rolling(pd.rolling_var)
+rolling_skew = wrap_rolling(pd.rolling_skew)
+rolling_kurt = wrap_rolling(pd.rolling_kurt)
+rolling_quantile = wrap_rolling(pd.rolling_quantile)
+rolling_apply = wrap_rolling(pd.rolling_apply)
+rolling_window = wrap_rolling(pd.rolling_window)


Test patch:
diff --git a/dask/dataframe/tests/test_rolling.py b/dask/dataframe/tests/test_rolling.py
new file mode 100644
--- /dev/null
+++ b/dask/dataframe/tests/test_rolling.py
@@ -0,0 +1,59 @@
+import pandas as pd
+import pandas.util.testing as tm
+import numpy as np
+
+import dask.dataframe as dd
+from dask.async import get_sync
+from dask.utils import raises
+
+
+def eq(p, d):
+    if isinstance(d, dd.DataFrame):
+        tm.assert_frame_equal(p, d.compute(get=get_sync))
+    else:
+        tm.assert_series_equal(p, d.compute(get=get_sync))
+
+
+def rolling_tests(p, d):
+    eq(pd.rolling_count(p, 3), dd.rolling_count(d, 3))
+    eq(pd.rolling_sum(p, 3), dd.rolling_sum(d, 3))
+    eq(pd.rolling_mean(p, 3), dd.rolling_mean(d, 3))
+    eq(pd.rolling_median(p, 3), dd.rolling_median(d, 3))
+    eq(pd.rolling_min(p, 3), dd.rolling_min(d, 3))
+    eq(pd.rolling_max(p, 3), dd.rolling_max(d, 3))
+    eq(pd.rolling_std(p, 3), dd.rolling_std(d, 3))
+    eq(pd.rolling_var(p, 3), dd.rolling_var(d, 3))
+    eq(pd.rolling_skew(p, 3), dd.rolling_skew(d, 3))
+    eq(pd.rolling_kurt(p, 3), dd.rolling_kurt(d, 3))
+    eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
+    mad = lambda x: np.fabs(x - x.mean()).mean()
+    eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
+    eq(pd.rolling_window(p, 3, 'boxcar'), dd.rolling_window(d, 3, 'boxcar'))
+    # Test with edge-case window sizes
+    eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
+    eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
+    # Test with kwargs
+    eq(pd.rolling_sum(p, 3, min_periods=3), dd.rolling_sum(d, 3, min_periods=3))
+
+
+def test_rolling_series():
+    ts = pd.Series(np.random.randn(25).cumsum())
+    dts = dd.from_pandas(ts, 3)
+    rolling_tests(ts, dts)
+
+
+def test_rolling_dataframe():
+    df = pd.DataFrame({'a': np.random.randn(25).cumsum(),
+                       'b': np.random.randn(25).cumsum()})
+    ddf = dd.from_pandas(df, 3)
+    rolling_tests(df, ddf)
+
+
+def test_raises():
+    df = pd.DataFrame({'a': np.random.randn(25).cumsum(),
+                       'b': np.random.randn(25).cumsum()})
+    ddf = dd.from_pandas(df, 3)
+    assert raises(TypeError, lambda: dd.rolling_mean(ddf, 1.5))
+    assert raises(ValueError, lambda: dd.rolling_mean(ddf, -1))
+    assert raises(NotImplementedError, lambda: dd.rolling_mean(ddf, 3, freq=2))
+    assert raises(NotImplementedError, lambda: dd.rolling_mean(ddf, 3, how='min'))