Repository: pandas-dev/pandas

Issue:
Subtracting two data time series with NaT yields Overflow error
#### Problem description

If I have a series of DateTime values and I want to subtract another series with DateTime value which contains NaT, in some case I get OverflowError. 

#### Code Sample, a copy-pastable example if possible

```python
s1 = pd.Series([pd.to_datetime('1969-12-31')])
s2 = pd.Series([pd.to_datetime('NaT')])
s1 - s2
```
#### Expected Output

```
0   NaT
dtype: timedelta64[ns]
```

#### OverflowError

<details>

OverflowError                             Traceback (most recent call last)
<ipython-input-64-8dfa2778fb13> in <module>
      1 s1 = pd.Series([pd.to_datetime('1969-12-31')])
      2 s2 = pd.Series([pd.to_datetime('NaT')])
----> 3 s1-s2

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/ops.py in wrapper(left, right)
   1550             # test_dt64_series_add_intlike, which the index dispatching handles
   1551             # specifically.
-> 1552             result = dispatch_to_index_op(op, left, right, pd.DatetimeIndex)
   1553             return construct_result(left, result,
   1554                                     index=left.index, name=res_name,

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/ops.py in dispatch_to_index_op(op, left, right, index_class)
   1189         left_idx = left_idx._shallow_copy(freq=None)
   1190     try:
-> 1191         result = op(left_idx, right)
   1192     except NullFrequencyError:
   1193         # DatetimeIndex and TimedeltaIndex with freq == None raise ValueError

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/ops.py in wrapper(left, right)
   1550             # test_dt64_series_add_intlike, which the index dispatching handles
   1551             # specifically.
-> 1552             result = dispatch_to_index_op(op, left, right, pd.DatetimeIndex)
   1553             return construct_result(left, result,
   1554                                     index=left.index, name=res_name,

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/ops.py in dispatch_to_index_op(op, left, right, index_class)
   1189         left_idx = left_idx._shallow_copy(freq=None)
   1190     try:
-> 1191         result = op(left_idx, right)
   1192     except NullFrequencyError:
   1193         # DatetimeIndex and TimedeltaIndex with freq == None raise ValueError

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/ops.py in rsub(left, right)
    146 
    147 def rsub(left, right):
--> 148     return right - left
    149 
    150 

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/indexes/datetimelike.py in __sub__(self, other)
    499         def __sub__(self, other):
    500             # dispatch to ExtensionArray implementation
--> 501             result = self._data.__sub__(maybe_unwrap_index(other))
    502             return wrap_arithmetic_op(self, other, result)
    503 

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/arrays/datetimelike.py in __sub__(self, other)
   1273         elif is_datetime64_dtype(other) or is_datetime64tz_dtype(other):
   1274             # DatetimeIndex, ndarray[datetime64]
-> 1275             result = self._sub_datetime_arraylike(other)
   1276         elif is_period_dtype(other):
   1277             # PeriodIndex

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/arrays/datetimes.py in _sub_datetime_arraylike(self, other)
    722         other_i8 = other.asi8
    723         new_values = checked_add_with_arr(self_i8, -other_i8,
--> 724                                           arr_mask=self._isnan)
    725         if self._hasnans or other._hasnans:
    726             mask = (self._isnan) | (other._isnan)

/opt/anaconda3/envs/calf2cow/lib/python3.6/site-packages/pandas/core/algorithms.py in checked_add_with_arr(arr, b, arr_mask, b_mask)
    936 
    937     if to_raise:
--> 938         raise OverflowError("Overflow in int64 addition")
    939     return arr + b
    940 

OverflowError: Overflow in int64 addition
</details>

#### Details

I think the problem is not masking NaT values in the second series when checking overflow in function `_sub_datetime_arraylike(self, other)`  in `pandas/core/arrays/datetimes.py`. As a result, if the date is internally represented as a negative integer, the subtraction overflows. However, if the date is positive of NaT is in the first series, everything works fine.

#### Other examples:

```python
s1 = pd.Series([pd.to_datetime('1970-01-01')])
s2 = pd.Series([pd.to_datetime('NaT')])
print(s1 - s2)

s1 = pd.Series([pd.to_datetime('1969-12-31')])
s2 = pd.Series([pd.to_datetime('NaT')])
print(s2 - s1)

print(pd.to_datetime('1969-12-31') - pd.to_datetime('NaT'))
```
Output

```
0   NaT
dtype: timedelta64[ns]
0   NaT
dtype: timedelta64[ns]
NaT
```




Discussion:
Yeah, looks like this broke from `0.23.4` - cc @jbrockmendel 

Resolving patch:
diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -720,11 +720,11 @@ def _sub_datetime_arraylike(self, other):
 
         self_i8 = self.asi8
         other_i8 = other.asi8
+        arr_mask = self._isnan | other._isnan
         new_values = checked_add_with_arr(self_i8, -other_i8,
-                                          arr_mask=self._isnan)
+                                          arr_mask=arr_mask)
         if self._hasnans or other._hasnans:
-            mask = (self._isnan) | (other._isnan)
-            new_values[mask] = iNaT
+            new_values[arr_mask] = iNaT
         return new_values.view('timedelta64[ns]')
 
     def _add_offset(self, offset):
diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py
--- a/pandas/core/arrays/timedeltas.py
+++ b/pandas/core/arrays/timedeltas.py
@@ -190,6 +190,8 @@ def __init__(self, values, dtype=_TD_DTYPE, freq=None, copy=False):
                 "ndarray, or Series or Index containing one of those."
             )
             raise ValueError(msg.format(type(values).__name__))
+        if values.ndim != 1:
+            raise ValueError("Only 1-dimensional input arrays are supported.")
 
         if values.dtype == 'i8':
             # for compat with datetime/timedelta/period shared methods,
@@ -945,6 +947,9 @@ def sequence_to_td64ns(data, copy=False, unit="ns", errors="raise"):
                         .format(dtype=data.dtype))
 
     data = np.array(data, copy=copy)
+    if data.ndim != 1:
+        raise ValueError("Only 1-dimensional input arrays are supported.")
+
     assert data.dtype == 'm8[ns]', data
     return data, inferred_freq
 
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -665,7 +665,8 @@ def __array_wrap__(self, result, context=None):
         """
         Gets called after a ufunc.
         """
-        if is_bool_dtype(result):
+        result = lib.item_from_zerodim(result)
+        if is_bool_dtype(result) or lib.is_scalar(result):
             return result
 
         attrs = self._get_attributes_dict()


Test patch:
diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py
--- a/pandas/tests/arithmetic/test_datetime64.py
+++ b/pandas/tests/arithmetic/test_datetime64.py
@@ -1440,6 +1440,20 @@ def test_dt64arr_add_sub_offset_ndarray(self, tz_naive_fixture,
 class TestDatetime64OverflowHandling(object):
     # TODO: box + de-duplicate
 
+    def test_dt64_overflow_masking(self, box_with_array):
+        # GH#25317
+        left = Series([Timestamp('1969-12-31')])
+        right = Series([NaT])
+
+        left = tm.box_expected(left, box_with_array)
+        right = tm.box_expected(right, box_with_array)
+
+        expected = TimedeltaIndex([NaT])
+        expected = tm.box_expected(expected, box_with_array)
+
+        result = left - right
+        tm.assert_equal(result, expected)
+
     def test_dt64_series_arith_overflow(self):
         # GH#12534, fixed by GH#19024
         dt = pd.Timestamp('1700-01-31')
diff --git a/pandas/tests/arrays/test_timedeltas.py b/pandas/tests/arrays/test_timedeltas.py
--- a/pandas/tests/arrays/test_timedeltas.py
+++ b/pandas/tests/arrays/test_timedeltas.py
@@ -9,6 +9,18 @@
 
 
 class TestTimedeltaArrayConstructor(object):
+    def test_only_1dim_accepted(self):
+        # GH#25282
+        arr = np.array([0, 1, 2, 3], dtype='m8[h]').astype('m8[ns]')
+
+        with pytest.raises(ValueError, match="Only 1-dimensional"):
+            # 2-dim
+            TimedeltaArray(arr.reshape(2, 2))
+
+        with pytest.raises(ValueError, match="Only 1-dimensional"):
+            # 0-dim
+            TimedeltaArray(arr[[0]].squeeze())
+
     def test_freq_validation(self):
         # ensure that the public constructor cannot create an invalid instance
         arr = np.array([0, 0, 1], dtype=np.int64) * 3600 * 10**9
@@ -51,6 +63,16 @@ def test_copy(self):
 
 
 class TestTimedeltaArray(object):
+    def test_np_sum(self):
+        # GH#25282
+        vals = np.arange(5, dtype=np.int64).view('m8[h]').astype('m8[ns]')
+        arr = TimedeltaArray(vals)
+        result = np.sum(arr)
+        assert result == vals.sum()
+
+        result = np.sum(pd.TimedeltaIndex(arr))
+        assert result == vals.sum()
+
     def test_from_sequence_dtype(self):
         msg = "dtype .*object.* cannot be converted to timedelta64"
         with pytest.raises(ValueError, match=msg):