Repository: pandas-dev/pandas
Pull request: implement assert_tzawareness_compat for DatetimeIndex
closes #18162 

ATM comparing tzaware `DatetimeIndex` with tznaive DTI fails to raise.  This PR implements `_assert_tzawareness_compa`t (which currently exists in `_Timestamp`) in `DatetimeIndex` to fix that.

That in turn causes breakage in `Series.replace`.  There's a small edit in `core.internals` to fix that, but I'm not sure that's the best way to make that work.

There is still one remaining test error in `tests.series.test_indexing.TestSeriesIndexing.test_getitem_setitem_datetimeindex`, which may either represent a bug or a case where tznaive/tzaware rules are relaxed.

```
N = 50
rng = date_range('1/1/1990', periods=N, freq='H', tz='US/Eastern')
ts = Series(np.random.randn(N), index=rng)

lb = "1990-01-01 04:00:00"
rb = "1990-01-01 07:00:00"

>>> result = ts[(ts.index >= lb) & (ts.index <= rb)]
TypeError: Cannot compare tz-naive and tz-aware datetime-like objects
```

Later in the same tests it tries to compare `ts.index` to naive `datetime` objects.  Is this a special case where we are intentionally less strict?

Problem statement:
DatetimeIndex comparison tzaware vs naive should raise
```
>>> dr = pd.date_range('2016-01-01', periods=6)
>>> dz = dr.tz_localize('US/Pacific')
>>> dr < dz
array([ True,  True,  True,  True,  True,  True], dtype=bool)
>>> dr[0] < dz[0]
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "pandas/_libs/tslib.pyx", line 1169, in pandas._libs.tslib._Timestamp.__richcmp__
  File "pandas/_libs/tslib.pyx", line 1230, in pandas._libs.tslib._Timestamp._assert_tzawareness_compat
TypeError: Cannot compare tz-naive and tz-aware timestamps
```

The vectorized comparison should raise too right?

Hints from issue discussion:
yes this should raise

Commit message: flake8 fixup

Patch to review:
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 111ba0c92aa9b..b5122eecd76af 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -11,7 +11,7 @@
 
 from pandas.core.dtypes.common import (
     _NS_DTYPE, _INT64_DTYPE,
-    is_object_dtype, is_datetime64_dtype,
+    is_object_dtype, is_datetime64_dtype, is_datetime64tz_dtype,
     is_datetimetz, is_dtype_equal,
     is_integer, is_float,
     is_integer_dtype,
@@ -105,8 +105,11 @@ def _dt_index_cmp(opname, cls, nat_result=False):
 
     def wrapper(self, other):
         func = getattr(super(DatetimeIndex, self), opname)
-        if (isinstance(other, datetime) or
-                isinstance(other, compat.string_types)):
+        if isinstance(other, (datetime, compat.string_types)):
+            if isinstance(other, compat.string_types):
+                # We need to convert in order to assert tzawareness
+                other = Timestamp(other)
+            self._assert_tzawareness_compat(other)
             other = _to_m8(other, tz=self.tz)
             result = func(other)
             if isna(other):
@@ -116,6 +119,7 @@ def wrapper(self, other):
                 other = DatetimeIndex(other)
             elif not isinstance(other, (np.ndarray, Index, ABCSeries)):
                 other = _ensure_datetime64(other)
+            self._assert_tzawareness_compat(other)
             result = func(np.asarray(other))
             result = _values_from_object(result)
 
@@ -652,6 +656,20 @@ def _simple_new(cls, values, name=None, freq=None, tz=None,
         result._reset_identity()
         return result
 
+    def _assert_tzawareness_compat(self, other):
+        # adapted from _Timestamp._assert_tzawareness_compat
+        other_tz = getattr(other, 'tzinfo', None)
+        if is_datetime64tz_dtype(other):
+            # Get tzinfo from Series dtype
+            other_tz = other.dtype.tz
+        if self.tz is None:
+            if other_tz is not None:
+                raise TypeError('Cannot compare tz-naive and tz-aware '
+                                'datetime-like objects.')
+        elif other_tz is None:
+            raise TypeError('Cannot compare tz-naive and tz-aware '
+                            'datetime-like objects')
+
     @property
     def tzinfo(self):
         """
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
index ca4984cc16673..9ea8f72ec9efb 100644
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -3486,10 +3486,19 @@ def replace_list(self, src_list, dest_list, inplace=False, regex=False,
         # figure out our mask a-priori to avoid repeated replacements
         values = self.as_matrix()
 
-        def comp(s):
-            if isna(s):
-                return isna(values)
-            return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq)
+        if is_datetime64tz_dtype(self):
+            # need to be careful not to compare tznaive to tzaware
+            def comp(s):
+                if isna(s):
+                    return isna(values)
+                return _maybe_compare(values, s, operator.eq)
+                # TODO: Is just not-converting `s` the right thing to do here?
+        else:
+            def comp(s):
+                if isna(s):
+                    return isna(values)
+                return _maybe_compare(values, getattr(s, 'asm8', s),
+                                      operator.eq)
 
         masks = [comp(s) for i, s in enumerate(src_list)]
 
diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py
index 076c3d6f25a89..5c10f4727596f 100644
--- a/pandas/tests/indexes/datetimes/test_datetime.py
+++ b/pandas/tests/indexes/datetimes/test_datetime.py
@@ -1,3 +1,5 @@
+import operator
+
 import pytest
 
 import numpy as np
@@ -248,6 +250,36 @@ def test_append_join_nondatetimeindex(self):
         # it works
         rng.join(idx, how='outer')
 
+    def test_comparison_tzawareness_compat(self):
+        # GH#18162
+        dr = pd.date_range('2016-01-01', periods=6)
+        dz = dr.tz_localize('US/Pacific')
+
+        ops = [operator.eq, operator.ne,
+               operator.gt, operator.ge,
+               operator.lt, operator.le]
+
+        for left, right in [(dr, dz), (dz, dr)]:
+            for op in ops:
+                with pytest.raises(TypeError):
+                    op(left, right)
+
+        # Check that there isn't a problem aware-aware and naive-naive do not
+        # raise
+        assert (dr == dr).all()
+        assert (dz == dz).all()
+
+        naive_series = pd.Series(dr)
+        aware_series = pd.Series(dz)
+        for op in ops:
+            with pytest.raises(TypeError):
+                op(dz, naive_series)
+            with pytest.raises(TypeError):
+                op(dr, aware_series)
+
+        # TODO: implement _assert_tzawareness_compat for the reverse
+        # comparison with the Series on the left-hand side
+
     def test_comparisons_coverage(self):
         rng = date_range('1/1/2000', periods=10)
 
diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py
index d141b378fe214..25966abb697de 100644
--- a/pandas/tests/series/test_indexing.py
+++ b/pandas/tests/series/test_indexing.py
@@ -449,6 +449,13 @@ def test_getitem_setitem_datetimeindex(self):
 
         lb = "1990-01-01 04:00:00"
         rb = "1990-01-01 07:00:00"
+        with pytest.raises(TypeError):
+            # tznaive vs tzaware comparison is invalid
+            # see GH#18376, GH#18162
+            ts[(ts.index >= lb) & (ts.index <= rb)]
+
+        lb = "1990-01-01 04:00:00-0500"
+        rb = "1990-01-01 07:00:00-0500"
         result = ts[(ts.index >= lb) & (ts.index <= rb)]
         expected = ts[4:8]
         assert_series_equal(result, expected)
@@ -474,6 +481,13 @@ def test_getitem_setitem_datetimeindex(self):
 
         lb = datetime(1990, 1, 1, 4)
         rb = datetime(1990, 1, 1, 7)
+        with pytest.raises(TypeError):
+            # tznaive vs tzaware comparison is invalid
+            # see GH#18376, GH#18162
+            ts[(ts.index >= lb) & (ts.index <= rb)]
+
+        lb = rng.tzinfo.localize(datetime(1990, 1, 1, 4))
+        rb = rng.tzinfo.localize(datetime(1990, 1, 1, 7))
         result = ts[(ts.index >= lb) & (ts.index <= rb)]
         expected = ts[4:8]
         assert_series_equal(result, expected)


Code review comments:
diff_hunk: @@ -248,6 +250,36 @@ def test_append_join_nondatetimeindex(self):
         # it works
         rng.join(idx, how='outer')
 
+    def test_comparison_tzawareness_compat(self):
+        # GH#18162
+        dr = pd.date_range('2016-01-01', periods=6)
+        dz = dr.tz_localize('US/Pacific')
+
+        ops = [operator.eq, operator.ne,
+               operator.gt, operator.ge,
+               operator.lt, operator.le]
+
+        for left, right in [(dr, dz), (dz, dr)]:
+            for op in ops:
+                with pytest.raises(TypeError):
+                    op(left, right)
+
+        # Check that there isn't a problem aware-aware and naive-naive do not
path: pandas/tests/indexes/datetimes/test_datetime.py
text: @user1:
these tests should be in series (the latter ones). alternatively a better place might be tin test_base for these, where we already handle series & index tests.