Repository: pandas-dev/pandas

Issue:
BLD: numpy master changes breaking
(24 hrs ago) good build: https://travis-ci.org/pydata/pandas/jobs/102356098: 1.11.0.dev0+51d2ecd
(1 hr ago) breaking lots of things: https://travis-ci.org/pydata/pandas/jobs/102596904: 1.11.0.dev0+aa6335c

@shoyer IIRC a couple of your PR's were merged in the last day.

here's an example:

```
======================================================================
FAIL: test_coercion_with_setitem_and_series (pandas.tests.test_indexing.TestSeriesNoneCoercion)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/build/pydata/pandas/pandas/tests/test_indexing.py", line 5247, in test_coercion_with_setitem_and_series
    expected_series.values, strict_nan=True)
  File "/home/travis/build/pydata/pandas/pandas/util/testing.py", line 866, in assert_numpy_array_equal
    raise_assert_detail(obj, msg, left, right)
  File "/home/travis/build/pydata/pandas/pandas/util/testing.py", line 825, in raise_assert_detail
    raise AssertionError(msg)
AssertionError: numpy array are different
numpy array values are different (33.33333 %)
[left]:  [NaT, 2000-01-02T00:00:00.000000000+0000, 2000-01-03T00:00:00.000000000+0000]
[right]: [NaT, 2000-01-02T00:00:00.000000000+0000, 2000-01-03T00:00:00.000000000+0000]
```



Discussion:
my guess is that we now need to compare `M8[ns]` by `.view('i8')` then comparing as the `NaT` will then compare equal.

Yep, if this is datetime64 related it's my fault :). I though we were already using .view('i8') before making comparisons but I guess I was wrong. If so (especially if this breaks user facing stuff) we may need to hold off on these numpy fixes for a little longer (deprecation cycle?). Sigh...

On Fri, Jan 15, 2016 at 7:27 AM, Jeff Reback notifications@github.com
wrote:

> ## my guess is that we now need to compare `M8[ns]` by `.view('i8')` then comparing as the `NaT` will then compare equal.
> 
> Reply to this email directly or view it on GitHub:
> https://github.com/pydata/pandas/issues/12049#issuecomment-171991205

we can adapt. do you want me to open an issue on numpy?

```
In [1]: np.__version__
Out[1]: '1.10.2'

In [5]: arr = np.array([np.nan])

In [6]: np.array_equal(arr,arr)
Out[6]: False

In [7]: arr = np.array([np.datetime64('NaT')])

In [8]: np.array_equal(arr,arr)
Out[8]: True
```

ok so your change makes sense

```
In [1]: np.__version__
Out[1]: '1.11.0.dev0+aa6335c'

In [2]: In [5]: arr = np.array([np.nan])

In [7]: arr = np.array([np.nan])

In [8]: np.array_equal(arr,arr)
Out[8]: False

In [9]: arr = np.array([np.datetime64('NaT')])

In [10]: np.array_equal(arr,arr)
Out[10]: False
```

Actually, looks like we might just be able to drop our `assert_numpy_array_equivalent` entirely in favor of numpy's `assert_array_equal`, which has supported NaN equality since at least August 2011 (numpy 1.7, I think):
https://github.com/numpy/numpy/commit/67ece6bdd2b35d011893e78154dbff6ab51c7d35

Unfortunately, this bug exists in our `array_equivalent` utility function, which we use for the `equals` method. This means that with this change datetime64 equality checks involving NaT will be broken. As much as I would love to just roll out the NumPy fix, doing it like this will assuredly result in unhappy users and unnecessary aggravation when sanity checks and test suites fail.

To fix this, I propose:
- We roll back the NaT comparison fix in NumPy for now, issuing a deprecation warning instead for a numpy release or two.
- We add the fix to `array_equivalent` in pandas, doing the appropriate cast to int64 to avoid needing to catch the deprecation warning (which can have performance consequences).

yep, prob need a special case for `M8/m8` (alternatively we could have a conditional check on the numpy version).

Ok, lmk if you need anything on the numpy side.


Resolving patch:
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -379,12 +379,13 @@ def array_equivalent(left, right, strict_nan=False):
     """
 
     left, right = np.asarray(left), np.asarray(right)
+
+    # shape compat
     if left.shape != right.shape:
         return False
 
     # Object arrays can contain None, NaN and NaT.
-    if (issubclass(left.dtype.type, np.object_) or
-            issubclass(right.dtype.type, np.object_)):
+    if is_object_dtype(left) or is_object_dtype(right):
 
         if not strict_nan:
             # pd.isnull considers NaN and None to be equivalent.
@@ -405,13 +406,21 @@ def array_equivalent(left, right, strict_nan=False):
         return True
 
     # NaNs can occur in float and complex arrays.
-    if issubclass(left.dtype.type, (np.floating, np.complexfloating)):
+    if is_float_dtype(left) or is_complex_dtype(left):
         return ((left == right) | (np.isnan(left) & np.isnan(right))).all()
 
     # numpy will will not allow this type of datetimelike vs integer comparison
     elif is_datetimelike_v_numeric(left, right):
         return False
 
+    # M8/m8
+    elif needs_i8_conversion(left) and needs_i8_conversion(right):
+        if not is_dtype_equal(left.dtype, right.dtype):
+            return False
+
+        left = left.view('i8')
+        right = right.view('i8')
+
     # NaNs cannot occur otherwise.
     return np.array_equal(left, right)
 


Test patch:
diff --git a/pandas/tests/test_common.py b/pandas/tests/test_common.py
--- a/pandas/tests/test_common.py
+++ b/pandas/tests/test_common.py
@@ -8,7 +8,9 @@
 import numpy as np
 import pandas as pd
 from pandas.tslib import iNaT, NaT
-from pandas import Series, DataFrame, date_range, DatetimeIndex, Timestamp, Float64Index
+from pandas import (Series, DataFrame, date_range,
+                    DatetimeIndex, TimedeltaIndex,
+                    Timestamp, Float64Index)
 from pandas import compat
 from pandas.compat import range, long, lrange, lmap, u
 from pandas.core.common import notnull, isnull, array_equivalent
@@ -322,20 +324,40 @@ def test_array_equivalent():
                             np.array([np.nan, 1, np.nan]))
     assert array_equivalent(np.array([np.nan, None], dtype='object'),
                             np.array([np.nan, None], dtype='object'))
-    assert array_equivalent(np.array([np.nan, 1+1j], dtype='complex'),
-                            np.array([np.nan, 1+1j], dtype='complex'))
-    assert not array_equivalent(np.array([np.nan, 1+1j], dtype='complex'),
-                                np.array([np.nan, 1+2j], dtype='complex'))
+    assert array_equivalent(np.array([np.nan, 1 + 1j], dtype='complex'),
+                            np.array([np.nan, 1 + 1j], dtype='complex'))
+    assert not array_equivalent(np.array([np.nan, 1 + 1j], dtype='complex'),
+                                np.array([np.nan, 1 + 2j], dtype='complex'))
     assert not array_equivalent(np.array([np.nan, 1, np.nan]),
                                 np.array([np.nan, 2, np.nan]))
-    assert not array_equivalent(np.array(['a', 'b', 'c', 'd']), np.array(['e', 'e']))
-    assert array_equivalent(Float64Index([0, np.nan]), Float64Index([0, np.nan]))
-    assert not array_equivalent(Float64Index([0, np.nan]), Float64Index([1, np.nan]))
-    assert array_equivalent(DatetimeIndex([0, np.nan]), DatetimeIndex([0, np.nan]))
-    assert not array_equivalent(DatetimeIndex([0, np.nan]), DatetimeIndex([1, np.nan]))
+    assert not array_equivalent(np.array(['a', 'b', 'c', 'd']),
+                                np.array(['e', 'e']))
+    assert array_equivalent(Float64Index([0, np.nan]),
+                            Float64Index([0, np.nan]))
+    assert not array_equivalent(Float64Index([0, np.nan]),
+                                Float64Index([1, np.nan]))
+    assert array_equivalent(DatetimeIndex([0, np.nan]),
+                            DatetimeIndex([0, np.nan]))
+    assert not array_equivalent(DatetimeIndex([0, np.nan]),
+                                DatetimeIndex([1, np.nan]))
+    assert array_equivalent(TimedeltaIndex([0, np.nan]),
+                            TimedeltaIndex([0, np.nan]))
+    assert not array_equivalent(TimedeltaIndex([0, np.nan]),
+                                TimedeltaIndex([1, np.nan]))
+    assert array_equivalent(DatetimeIndex([0, np.nan], tz='US/Eastern'),
+                            DatetimeIndex([0, np.nan], tz='US/Eastern'))
+    assert not array_equivalent(DatetimeIndex([0, np.nan], tz='US/Eastern'),
+                                DatetimeIndex([1, np.nan], tz='US/Eastern'))
+    assert not array_equivalent(DatetimeIndex([0, np.nan]),
+                                DatetimeIndex([0, np.nan], tz='US/Eastern'))
+    assert not array_equivalent(DatetimeIndex([0, np.nan], tz='CET'),
+                                DatetimeIndex([0, np.nan], tz='US/Eastern'))
+    assert not array_equivalent(DatetimeIndex([0, np.nan]),
+                                TimedeltaIndex([0, np.nan]))
+
 
 def test_datetimeindex_from_empty_datetime64_array():
-    for unit in [ 'ms', 'us', 'ns' ]:
+    for unit in ['ms', 'us', 'ns']:
         idx = DatetimeIndex(np.array([], dtype='datetime64[%s]' % unit))
         assert(len(idx) == 0)
 
diff --git a/pandas/tseries/tests/test_tslib.py b/pandas/tseries/tests/test_tslib.py
--- a/pandas/tseries/tests/test_tslib.py
+++ b/pandas/tseries/tests/test_tslib.py
@@ -399,8 +399,10 @@ def test_asm8(self):
             1000,
         ]
         for n in ns:
-            self.assertEqual(Timestamp(n).asm8, np.datetime64(n, 'ns'), n)
-        self.assertEqual(Timestamp('nat').asm8, np.datetime64('nat', 'ns'))
+            self.assertEqual(Timestamp(n).asm8.view('i8'),
+                             np.datetime64(n, 'ns').view('i8'), n)
+        self.assertEqual(Timestamp('nat').asm8.view('i8'),
+                         np.datetime64('nat', 'ns').view('i8'))
 
     def test_fields(self):
 
@@ -752,13 +754,11 @@ def test_coercing_dates_outside_of_datetime64_ns_bounds(self):
                 np.array([invalid_date], dtype='object'),
                 errors='raise',
             )
-            self.assertTrue(
-                np.array_equal(
-                    tslib.array_to_datetime(
-                        np.array([invalid_date], dtype='object'), errors='coerce',
-                    ),
-                    np.array([tslib.iNaT], dtype='M8[ns]')
-                )
+            self.assert_numpy_array_equal(
+                tslib.array_to_datetime(
+                    np.array([invalid_date], dtype='object'),
+                    errors='coerce'),
+                np.array([tslib.iNaT], dtype='M8[ns]')
             )
 
         arr = np.array(['1/1/1000', '1/1/2000'], dtype=object)