Repository: dask/dask

Issue:
Resample fails on freqs > day
Resample currently fails on certain cases. These are:
- Empty bins that are between two partitions. Pandas fills these with NaN (or as specified by a kwarg), these bins will just not be there in dask.
- Weirdness from the behavior of closed='right' and label='right' kwargs. These are also the defaults for resample('W'). This can cause repeat bins, as we didn't actually split on the partition appropriately.



Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -834,31 +834,42 @@ def resample(self, rule, how='mean', axis=0, fill_method=None, closed=None,
         pandas.Series.resample
         """
 
-        divs = pd.Series(range(len(self.divisions)), index=self.divisions)
-        temp = divs.resample(rule, how='count', axis=axis, fill_method=fill_method,
-                          closed=closed, label=label, convention=convention,
-                          kind=kind, loffset=loffset, limit=limit, base=base)
-        newdivs = temp.loc[temp > 0].index.tolist()
-        if newdivs[-1] < self.divisions[-1]:
-            newdivs.append(self.divisions[-1])
-        if newdivs[0] > self.divisions[0]:
-            newdivs.insert(0, self.divisions[0])
-
-        day_nanos = pd.datetools.Day().nanos
-
+        # Validate Inputs
         rule = pd.datetools.to_offset(rule)
-
+        day_nanos = pd.datetools.Day().nanos
         if getattr(rule, 'nanos', None) and day_nanos % rule.nanos:
             raise NotImplementedError('Resampling frequency %s that does'
                                       ' not evenly divide a day is not '
                                       'implemented' % rule)
-
-        return map_partitions(pd.Series.resample, self.name,
-                self.repartition(newdivs, force=True),
-                rule=rule, how=how, axis=axis,
-                fill_method=fill_method, closed=closed, label=label,
-                convention=convention, kind=kind, loffset=loffset, limit=limit,
-                base=base)
+        kwargs = {'fill_method': fill_method, 'limit': limit,
+                  'loffset': loffset, 'base': base,
+                  'convention': convention != 'start', 'kind': kind}
+        err = ', '.join('`{0}`'.format(k) for (k, v) in kwargs.items() if v)
+        if err:
+            raise NotImplementedError('Keywords: ' + err)
+
+        # Create a grouper to determine closed and label conventions
+        newdivs, outdivs = _resample_bin_and_out_divs(self.divisions, rule,
+                                                      closed, label)
+
+        # Repartition divs into bins. These won't match labels after mapping
+        partitioned = self.repartition(newdivs, force=True)
+
+        kwargs = {'how': how, 'closed': closed, 'label': label}
+        name = tokenize(self, rule, kwargs)
+        dsk = partitioned.dask
+        def func(series, start, end, closed):
+            out = series.resample(rule, **kwargs)
+            return out.reindex(pd.date_range(start, end, freq=rule, closed=closed))
+
+        keys = partitioned._keys()
+        args = zip(keys, outdivs, outdivs[1:], ['left']*(len(keys)-1) + [None])
+        for i, (k, s, e, c) in enumerate(args):
+            dsk[(name, i)] = (func, k, s, e, c)
+
+        if how == 'ohlc':
+            return DataFrame(dsk, name, ['open', 'high', 'low', 'close'], outdivs)
+        return Series(dsk, name, self.name, outdivs)
 
     def __getitem__(self, key):
         if isinstance(key, Series) and self.divisions == key.divisions:
@@ -2072,6 +2083,46 @@ def pd_split(df, p, seed=0):
     return [df.iloc[index == i] for i in range(len(p))]
 
 
+def _resample_bin_and_out_divs(divisions, rule, closed, label):
+    rule = pd.datetools.to_offset(rule)
+    g = pd.TimeGrouper(rule, how='count', closed=closed, label=label)
+
+    # Determine bins to apply `how` to. Disregard labeling scheme.
+    divs = pd.Series(range(len(divisions)), index=divisions)
+    temp = divs.resample(rule, how='count', closed=closed, label='left')
+    tempdivs = temp.loc[temp > 0].index
+
+    # Cleanup closed == 'right' and label == 'right'
+    res = pd.offsets.Nano() if hasattr(rule, 'delta') else pd.offsets.Day()
+    if g.closed == 'right':
+        newdivs = tempdivs + res
+    else:
+        newdivs = tempdivs
+    if g.label == 'right':
+        outdivs = tempdivs + rule
+    else:
+        outdivs = tempdivs
+
+    newdivs = newdivs.tolist()
+    outdivs = outdivs.tolist()
+
+    # Adjust ends
+    if newdivs[0] < divisions[0]:
+        newdivs[0] = divisions[0]
+    if newdivs[-1] < divisions[-1]:
+        if len(newdivs) < len(divs):
+            setter = lambda a, val: a.append(val)
+        else:
+            setter = lambda a, val: a.__setitem__(-1, val)
+        setter(newdivs, divisions[-1])
+        if outdivs[-1] > divisions[-1]:
+            setter(outdivs, outdivs[-1])
+        elif outdivs[-1] < divisions[-1]:
+            setter(outdivs, temp.index[-1])
+
+    return tuple(map(pd.Timestamp, newdivs)), tuple(map(pd.Timestamp, outdivs))
+
+
 def repartition_divisions(a, b, name, out1, out2, force=False):
     """ dask graph to repartition dataframe by new divisions
 


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1776,58 +1776,39 @@ def test_dataframe_groupby_nunique_across_group_same_value():
     assert eq(s.groupby('strings')['data'].nunique(), expected)
 
 
-@pytest.mark.parametrize(['freq', 'how', 'npartitions', 'nskipped'],
-                         list(product(['30T', 'H', 'D'],
-                                      ['sum', 'mean', 'count', 'nunique'],
-                                      [2, 3],
-                                      [2, 3])))
-def test_series_resample(freq, how, npartitions, nskipped):
-    n = 24 * 60 * 3
-    index = pd.date_range(start='20120102', periods=n, freq='T').values
-    index = index[::nskipped]
-    s = pd.Series(np.arange(len(index), dtype='f8'), index=pd.Index(index))
-    expected = s.resample(freq, how=how)
-    ds = dd.from_pandas(s, npartitions=npartitions)
-
-    resampled = ds.resample(freq, how=how)
-    result = resampled.compute()
-    assert resampled.divisions[0] == result.index[0]
-    tm.assert_series_equal(result, expected, check_dtype=False)
-
-
-@pytest.mark.xfail(raises=NotImplementedError)
-@pytest.mark.parametrize(['freq', 'how', 'npartitions', 'nskipped'],
-                         list(product(['57T'],
-                                      ['sum', 'mean', 'count', 'nunique'],
-                                      [2, 3],
-                                      [2, 3])))
-def test_series_resample_failing(freq, how, npartitions, nskipped):
-    n = 24 * 60 * 3
-    index = pd.date_range(start='20120102', periods=n, freq='T').values
-    index = index[::nskipped]
-    s = pd.Series(np.arange(len(index), dtype='f8'), index=pd.Index(index))
-    expected = s.resample(freq, how=how)
-    ds = dd.from_pandas(s, npartitions=npartitions)
-
-    resampled = ds.resample(freq, how=how)
-    result = resampled.compute()
-    tm.assert_series_equal(result, expected, check_dtype=False)
-
-
-@pytest.mark.parametrize(['how', 'npartitions'],
-                         list(product(['sum', 'mean', 'count', 'nunique'],
-                                      [2, 5])))
-def test_series_resample_big_freq(how, npartitions):
-    from pandas.tseries.offsets import Week
-    freq = 'W'
-    index = pd.date_range('21-Jan-2013', '3-NOV-2014', freq=Week())
+@pytest.mark.parametrize(['npartitions', 'freq', 'closed', 'label'],
+                         list(product([2, 5], ['30T', 'h', 'd', 'w', 'M'],
+                                      ['right', 'left'], ['right', 'left'])))
+def test_series_resample(npartitions, freq, closed, label):
+    index = pd.date_range('1-1-2000', '2-15-2000', freq='h')
+    index = index.union(pd.date_range('4-15-2000', '5-15-2000', freq='h'))
     df = pd.Series(range(len(index)), index=index)
     ds = dd.from_pandas(df, npartitions=npartitions)
-
-    resampled = ds.resample(freq, how=how)
-    result = resampled.compute()
-    expected = df.resample(freq, how=how)
+    # Series output
+    result = ds.resample(freq, how='mean', closed=closed, label=label).compute()
+    expected = df.resample(freq, how='mean', closed=closed, label=label)
     tm.assert_series_equal(result, expected, check_dtype=False)
+    # Frame output
+    resampled = ds.resample(freq, how='ohlc', closed=closed, label=label)
+    divisions = resampled.divisions
+    result = resampled.compute()
+    expected = df.resample(freq, how='ohlc', closed=closed, label=label)
+    tm.assert_frame_equal(result, expected, check_dtype=False)
+    assert expected.index[0] == divisions[0]
+    assert expected.index[-1] == divisions[-1]
+
+
+def test_series_resample_not_implemented():
+    index = pd.date_range(start='20120102', periods=100, freq='T')
+    s = pd.Series(range(len(index)), index=index)
+    ds = dd.from_pandas(s, npartitions=5)
+    # Frequency doesn't evenly divide day
+    assert raises(NotImplementedError, lambda: ds.resample('57T'))
+    # Kwargs not implemented
+    kwargs = {'fill_method': 'bfill', 'limit': 2, 'loffset': 2, 'base': 2,
+              'convention': 'end', 'kind': 'period'}
+    for k, v in kwargs.items():
+        assert raises(NotImplementedError, lambda: ds.resample('6h', **{k: v}))
 
 
 def test_set_partition_2():