Repository: pandas-dev/pandas

Issue:
TST: failures with latest pytz (2014.3)
only shows on the numpy 1.9-dev build as this is the only one to pull a current version of pytz
Bug report for pytz

https://bugs.launchpad.net/pytz/+bug/1319939

```
INSTALLED VERSIONS
------------------
commit: 592a5379dea49f6e88e736a651bf82a00ea4fcc8
python: 2.7.3.final.0
python-bits: 64
OS: Linux
OS-release: 2.6.32-5-amd64
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8

pandas: None
nose: 1.3.3
Cython: 0.20.1
numpy: 1.9.0.dev-13b32e9
scipy: None
statsmodels: None
IPython: None
sphinx: None
patsy: None
scikits.timeseries: None
dateutil: 2.2
pytz: 2014.3
bottleneck: 0.9.0dev
tables: None
numexpr: None
matplotlib: None
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: None
bq: None
apiclient: None
rpy2: None
sqlalchemy: None
pymysql: None
psycopg2: None
```

```
======================================================================
FAIL: test_getitem_setitem_datetime_tz (pandas.tests.test_series.TestSeries)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/tests/test_series.py", line 4594, in test_getitem_setitem_datetime_tz
    assert_series_equal(result, ts)
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/util/testing.py", line 530, in assert_series_equal
    assert_almost_equal(left.values, right.values, check_less_precise)
  File "testing.pyx", line 58, in pandas._testing.assert_almost_equal (pandas/src/testing.c:2536)
  File "testing.pyx", line 82, in pandas._testing.assert_almost_equal (pandas/src/testing.c:1710)
AssertionError: Length of two iterators not the same: 51 != 50

----------------------------------------------------------------------
Ran 311 tests in 2.951s

FAILED (SKIP=6, failures=1)
(py2.7_bottleneck)jreback@sheep:~/venv/py2.7_bottleneck/pandas$ nosetests pandas//tseries/tests/test_timezones.py 
.................E.....F.......................F..................
======================================================================
ERROR: test_getitem_pydatetime_tz (pandas.tseries.tests.test_timezones.TestTimeZoneSupport)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/tseries/tests/test_timezones.py", line 648, in test_getitem_pydatetime_tz
    self.assertEqual(ts[time_pandas], ts[time_datetime])
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/core/series.py", line 475, in __getitem__
    result = self.index.get_value(self, key)
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/tseries/index.py", line 1263, in get_value
    return self.get_value_maybe_box(series, key)
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/tseries/index.py", line 1289, in get_value_maybe_box
    values = self._engine.get_value(_values_from_object(series), key)
  File "index.pyx", line 97, in pandas.index.IndexEngine.get_value (pandas/index.c:2987)
  File "index.pyx", line 105, in pandas.index.IndexEngine.get_value (pandas/index.c:2802)
  File "index.pyx", line 515, in pandas.index.DatetimeEngine.get_loc (pandas/index.c:9256)
KeyError: Timestamp('2012-12-24 17:07:00+0100', tz='Europe/Berlin')

======================================================================
FAIL: test_infer_tz (pandas.tseries.tests.test_timezones.TestTimeZoneSupport)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/tseries/tests/test_timezones.py", line 396, in test_infer_tz
    assert(tools._infer_tzinfo(start, end) is eastern)
AssertionError

======================================================================
FAIL: test_with_tz (pandas.tseries.tests.test_timezones.TestTimeZoneSupport)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/tseries/tests/test_timezones.py", line 321, in test_with_tz
    self.assertIs(central[0].tz, tz)
  File "/mnt/home/jreback/venv/py2.7_bottleneck/pandas/pandas/util/testing.py", line 94, in assertIs
    assert a is b, "%s: %r is not %r" % (msg.format(a,b), a, b)
AssertionError: : <DstTzInfo 'US/Central' CST-1 day, 18:00:00 STD> is not <DstTzInfo 'US/Central' LMT-1 day, 18:09:00 STD>

----------------------------------------------------------------------
Ran 66 tests in 0.488s

FAILED (errors=1, failures=2)
(py2.7_bottleneck)jreback@sheep:~/venv/py2.7_bottleneck/pandas$ 
```



Discussion:

Resolving patch:
diff --git a/pandas/tseries/index.py b/pandas/tseries/index.py
--- a/pandas/tseries/index.py
+++ b/pandas/tseries/index.py
@@ -257,7 +257,7 @@ def __new__(cls, data=None,
 
             if lib.is_string_array(values):
                 subarr = _str_to_dt_array(values, freq, dayfirst=dayfirst,
-                                        yearfirst=yearfirst)          
+                                        yearfirst=yearfirst)
             else:
                 try:
                     subarr = tools.to_datetime(data, box=False)
@@ -350,7 +350,9 @@ def _generate(cls, start, end, periods, name, offset,
                              'different timezones')
 
         inferred_tz = tools._maybe_get_tz(inferred_tz)
-        tz = tools._maybe_get_tz(tz)
+
+        # these may need to be localized
+        tz = tools._maybe_get_tz(tz, start or end)
 
         if tz is not None and inferred_tz is not None:
             if not inferred_tz == tz:
diff --git a/pandas/tseries/tools.py b/pandas/tseries/tools.py
--- a/pandas/tseries/tools.py
+++ b/pandas/tseries/tools.py
@@ -56,13 +56,19 @@ def _infer(a, b):
     return tz
 
 
-def _maybe_get_tz(tz):
+def _maybe_get_tz(tz, date=None):
     if isinstance(tz, compat.string_types):
         import pytz
         tz = pytz.timezone(tz)
     if com.is_integer(tz):
         import pytz
         tz = pytz.FixedOffset(tz / 60)
+
+    # localize and get the tz
+    if date is not None and tz is not None:
+        if date.tzinfo is not None and hasattr(tz,'localize'):
+            tz = tz.localize(date.replace(tzinfo=None)).tzinfo
+
     return tz
 
 def _guess_datetime_format(dt_str, dayfirst=False,


Test patch:
diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py
--- a/pandas/tests/test_series.py
+++ b/pandas/tests/test_series.py
@@ -4589,8 +4589,11 @@ def test_getitem_setitem_datetime_tz(self):
         assert_series_equal(result, ts)
 
         result = ts.copy()
-        result[datetime(1990, 1, 1, 3, tzinfo=tz('US/Central'))] = 0
-        result[datetime(1990, 1, 1, 3, tzinfo=tz('US/Central'))] = ts[4]
+
+        # comparison dates with datetime MUST be localized!
+        date = tz('US/Central').localize(datetime(1990, 1, 1, 3))
+        result[date] = 0
+        result[date] = ts[4]
         assert_series_equal(result, ts)
 
     def test_getitem_setitem_periodindex(self):
diff --git a/pandas/tseries/tests/test_timezones.py b/pandas/tseries/tests/test_timezones.py
--- a/pandas/tseries/tests/test_timezones.py
+++ b/pandas/tseries/tests/test_timezones.py
@@ -317,8 +317,11 @@ def test_with_tz(self):
 
         # normalized
         central = dr.tz_convert(tz)
+
+        # compare vs a localized tz
+        comp = tz.localize(dr[0].to_pydatetime().replace(tzinfo=None)).tzinfo
         self.assertIs(central.tz, tz)
-        self.assertIs(central[0].tz, tz)
+        self.assertIs(central[0].tz, comp)
 
         # datetimes with tzinfo set
         dr = bdate_range(datetime(2005, 1, 1, tzinfo=pytz.utc),
@@ -393,9 +396,9 @@ def test_infer_tz(self):
 
         start = eastern.localize(_start)
         end = eastern.localize(_end)
-        assert(tools._infer_tzinfo(start, end) is eastern)
-        assert(tools._infer_tzinfo(start, None) is eastern)
-        assert(tools._infer_tzinfo(None, end) is eastern)
+        assert(tools._infer_tzinfo(start, end) is eastern.localize(_start).tzinfo)
+        assert(tools._infer_tzinfo(start, None) is eastern.localize(_start).tzinfo)
+        assert(tools._infer_tzinfo(None, end) is eastern.localize(_end).tzinfo)
 
         start = utc.localize(_start)
         end = utc.localize(_end)
@@ -643,8 +646,7 @@ def test_getitem_pydatetime_tz(self):
                            tz='Europe/Berlin')
         ts = Series(index=index, data=index.hour)
         time_pandas = Timestamp('2012-12-24 17:00', tz='Europe/Berlin')
-        time_datetime = datetime(2012, 12, 24, 17, 0,
-                                 tzinfo=pytz.timezone('Europe/Berlin'))
+        time_datetime = pytz.timezone('Europe/Berlin').localize(datetime(2012, 12, 24, 17, 0))
         self.assertEqual(ts[time_pandas], ts[time_datetime])
 
     def test_index_drop_dont_lose_tz(self):
@@ -974,7 +976,7 @@ def test_tzaware_offset(self):
                 self.assert_(offset.equals(expected))
             offset = dates + timedelta(hours=5)
             self.assert_(offset.equals(expected))
-            
+
     def test_nat(self):
         # GH 5546
         dates = [NaT]