Repository: pandas-dev/pandas

Issue:
BUG: Resampling a TZ-aware Series fails if rule='W': "Values falls before first bin"
``` python
In [1]: import pandas as pd

In [2]: rng = pd.date_range("2013-04-01", "2013-05-01", tz='Europe/London', freq='H')

In [4]: series = pd.Series(index=rng)

In [6]: series.resample('W')
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-6-906374246edd> in <module>()
----> 1 series.resample('W')

/usr/local/lib/python2.7/dist-packages/pandas/core/generic.pyc in resample(self, rule, how, axis, fill_method, closed, label, convention, kind, loffset, limit, base)
   3003                               fill_method=fill_method, convention=convention,
   3004                               limit=limit, base=base)
-> 3005         return sampler.resample(self).__finalize__(self)
   3006 
   3007     def first(self, offset):

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in resample(self, obj)
     83 
     84         if isinstance(ax, DatetimeIndex):
---> 85             rs = self._resample_timestamps()
     86         elif isinstance(ax, PeriodIndex):
     87             offset = to_offset(self.freq)

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in _resample_timestamps(self, kind)
    273         axlabels = self.ax
    274 
--> 275         self._get_binner_for_resample(kind=kind)
    276         grouper = self.grouper
    277         binner = self.binner

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in _get_binner_for_resample(self, kind)
    121             kind = self.kind
    122         if kind is None or kind == 'timestamp':
--> 123             self.binner, bins, binlabels = self._get_time_bins(ax)
    124         elif kind == 'timedelta':
    125             self.binner, bins, binlabels = self._get_time_delta_bins(ax)

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in _get_time_bins(self, ax)
    182 
    183         # general version, knowing nothing about relative frequencies
--> 184         bins = lib.generate_bins_dt64(ax_values, bin_edges, self.closed, hasnans=ax.hasnans)
    185 
    186         if self.closed == 'right':

/usr/local/lib/python2.7/dist-packages/pandas/lib.so in pandas.lib.generate_bins_dt64 (pandas/lib.c:17825)()

ValueError: Values falls before first bin
```

Resampling to weekly works fine if we first remove the timezone:

``` python
In [7]: series.tz_localize(None).resample('W')
Out[7]: 
2013-04-07   NaN
2013-04-14   NaN
2013-04-21   NaN
2013-04-28   NaN
2013-05-05   NaN
Freq: W-SUN, dtype: float64
```

And resampling the TZ-aware Series to daily works fine:

``` python
In [8]: series.resample('D')
Out[8]: 
2013-04-01 00:00:00+01:00   NaN
2013-04-02 00:00:00+01:00   NaN
2013-04-03 00:00:00+01:00   NaN
2013-04-04 00:00:00+01:00   NaN
2013-04-05 00:00:00+01:00   NaN
2013-04-06 00:00:00+01:00   NaN
2013-04-07 00:00:00+01:00   NaN
2013-04-08 00:00:00+01:00   NaN
2013-04-09 00:00:00+01:00   NaN
2013-04-10 00:00:00+01:00   NaN
2013-04-11 00:00:00+01:00   NaN
2013-04-12 00:00:00+01:00   NaN
2013-04-13 00:00:00+01:00   NaN
2013-04-14 00:00:00+01:00   NaN
2013-04-15 00:00:00+01:00   NaN
2013-04-16 00:00:00+01:00   NaN
2013-04-17 00:00:00+01:00   NaN
2013-04-18 00:00:00+01:00   NaN
2013-04-19 00:00:00+01:00   NaN
2013-04-20 00:00:00+01:00   NaN
2013-04-21 00:00:00+01:00   NaN
2013-04-22 00:00:00+01:00   NaN
2013-04-23 00:00:00+01:00   NaN
2013-04-24 00:00:00+01:00   NaN
2013-04-25 00:00:00+01:00   NaN
2013-04-26 00:00:00+01:00   NaN
2013-04-27 00:00:00+01:00   NaN
2013-04-28 00:00:00+01:00   NaN
2013-04-29 00:00:00+01:00   NaN
2013-04-30 00:00:00+01:00   NaN
2013-05-01 00:00:00+01:00   NaN
Freq: D, dtype: float64
```

This issue is similar (but not identical) to #1459 and #8941

My install versions:

``` python
In [2]: pd.show_versions()

INSTALLED VERSIONS
------------------
commit: None
python: 2.7.8.final.0
python-bits: 64
OS: Linux
OS-release: 3.16.0-28-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_GB.UTF-8

pandas: 0.15.2
nose: 1.3.4
Cython: 0.21.1
numpy: 1.9.1
scipy: 0.14.0
statsmodels: 0.6.1
IPython: 2.3.1
sphinx: 1.2.3
patsy: 0.3.0
dateutil: 2.3
pytz: 2014.10
bottleneck: None
tables: 3.1.1
numexpr: 2.4
matplotlib: 1.4.2
openpyxl: 1.7.0
xlrd: 0.9.2
xlwt: 0.7.5
xlsxwriter: None
lxml: 3.3.6
bs4: 4.3.2
html5lib: 0.999
httplib2: 0.9
apiclient: None
rpy2: 2.3.8
sqlalchemy: None
pymysql: None
psycopg2: 2.5.3 (dt dec pq3 ext)
```

(as always, I must applaud the Pandas dev team.  It's an awesome animal!)



Discussion:
This may be fixed in #5172 or maybe not (those are specific DST issues, but I saw some things in general that might overall fix it).  I will see if fixing those cases fixes this as well.

I have just checked this issue on Pandas 0.16.1 and I can confirm that this issue is still present (which I think we knew, but I just wanted to confirm!) Although the line numbers in the exception have changed a bit:

``` python
In [1]: import pandas as pd

In [2]: rng = pd.date_range("2013-04-01", "2013-05-01", tz='Europe/London', freq='H')

In [3]: series = pd.Series(index=rng)

In [4]: series.resample('W')
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-4-906374246edd> in <module>()
----> 1 series.resample('W')

/usr/local/lib/python2.7/dist-packages/pandas/core/generic.pyc in resample(self, rule, how, axis, fill_method, closed, label, convention, kind, loffset, limit, base)
   3195                               fill_method=fill_method, convention=convention,
   3196                               limit=limit, base=base)
-> 3197         return sampler.resample(self).__finalize__(self)
   3198 
   3199     def first(self, offset):

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in resample(self, obj)
     83 
     84         if isinstance(ax, DatetimeIndex):
---> 85             rs = self._resample_timestamps()
     86         elif isinstance(ax, PeriodIndex):
     87             offset = to_offset(self.freq)

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in _resample_timestamps(self, kind)
    273         axlabels = self.ax
    274 
--> 275         self._get_binner_for_resample(kind=kind)
    276         grouper = self.grouper
    277         binner = self.binner

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in _get_binner_for_resample(self, kind)
    121             kind = self.kind
    122         if kind is None or kind == 'timestamp':
--> 123             self.binner, bins, binlabels = self._get_time_bins(ax)
    124         elif kind == 'timedelta':
    125             self.binner, bins, binlabels = self._get_time_delta_bins(ax)

/usr/local/lib/python2.7/dist-packages/pandas/tseries/resample.pyc in _get_time_bins(self, ax)
    182 
    183         # general version, knowing nothing about relative frequencies
--> 184         bins = lib.generate_bins_dt64(ax_values, bin_edges, self.closed, hasnans=ax.hasnans)
    185 
    186         if self.closed == 'right':

pandas/lib.pyx in pandas.lib.generate_bins_dt64 (pandas/lib.c:18952)()

ValueError: Values falls before first bin
```

Yes, this is outstanding.  It was a different type of issue that wasn't easily fixed at the time.


Resolving patch:
diff --git a/pandas/core/resample.py b/pandas/core/resample.py
--- a/pandas/core/resample.py
+++ b/pandas/core/resample.py
@@ -16,7 +16,8 @@
 from pandas.tseries.frequencies import to_offset, is_subperiod, is_superperiod
 from pandas.core.indexes.datetimes import DatetimeIndex, date_range
 from pandas.core.indexes.timedeltas import TimedeltaIndex
-from pandas.tseries.offsets import DateOffset, Tick, Day, delta_to_nanoseconds
+from pandas.tseries.offsets import (DateOffset, Tick, Day,
+                                    delta_to_nanoseconds, Nano)
 from pandas.core.indexes.period import PeriodIndex
 from pandas.errors import AbstractMethodError
 import pandas.core.algorithms as algos
@@ -1395,18 +1396,21 @@ def _get_time_bins(self, ax):
     def _adjust_bin_edges(self, binner, ax_values):
         # Some hacks for > daily data, see #1471, #1458, #1483
 
-        bin_edges = binner.asi8
-
         if self.freq != 'D' and is_superperiod(self.freq, 'D'):
-            day_nanos = delta_to_nanoseconds(timedelta(1))
             if self.closed == 'right':
-                bin_edges = bin_edges + day_nanos - 1
+                # GH 21459, GH 9119: Adjust the bins relative to the wall time
+                bin_edges = binner.tz_localize(None)
+                bin_edges = bin_edges + timedelta(1) - Nano(1)
+                bin_edges = bin_edges.tz_localize(binner.tz).asi8
+            else:
+                bin_edges = binner.asi8
 
             # intraday values on last day
             if bin_edges[-2] > ax_values.max():
                 bin_edges = bin_edges[:-1]
                 binner = binner[:-1]
-
+        else:
+            bin_edges = binner.asi8
         return binner, bin_edges
 
     def _get_time_delta_bins(self, ax):


Test patch:
diff --git a/pandas/tests/test_resample.py b/pandas/tests/test_resample.py
--- a/pandas/tests/test_resample.py
+++ b/pandas/tests/test_resample.py
@@ -2114,6 +2114,28 @@ def test_downsample_across_dst(self):
                                            freq='H'))
         tm.assert_series_equal(result, expected)
 
+    def test_downsample_across_dst_weekly(self):
+        # GH 9119, GH 21459
+        df = DataFrame(index=DatetimeIndex([
+            '2017-03-25', '2017-03-26', '2017-03-27',
+            '2017-03-28', '2017-03-29'
+        ], tz='Europe/Amsterdam'),
+            data=[11, 12, 13, 14, 15])
+        result = df.resample('1W').sum()
+        expected = DataFrame([23, 42], index=pd.DatetimeIndex([
+            '2017-03-26', '2017-04-02'
+        ], tz='Europe/Amsterdam'))
+        tm.assert_frame_equal(result, expected)
+
+        idx = pd.date_range("2013-04-01", "2013-05-01", tz='Europe/London',
+                            freq='H')
+        s = Series(index=idx)
+        result = s.resample('W').mean()
+        expected = Series(index=pd.date_range(
+            '2013-04-07', freq='W', periods=5, tz='Europe/London'
+        ))
+        tm.assert_series_equal(result, expected)
+
     def test_resample_with_nat(self):
         # GH 13020
         index = DatetimeIndex([pd.NaT,