Repository: pandas-dev/pandas

Issue:
BUG: pd.to_timedelta raises instead of returning NaT
```
In [1]: Series(['2011-01-01', np.nan]).apply(pd.to_datetime)
Out[1]: 
0   2011-01-01 00:00:00
1                   NaT
dtype: datetime64[ns]

In [2]: Series(['00:00:01', np.nan]).apply(pd.to_timedelta)
AssertionError: Invalid type for timedelta scalar: <type 'float'>
```

May not be a trivial fix. So far I have seen that inserting np.nan and pd.tslib.iNaT into the result turns the dtype to object, which is no good.



Discussion:
That said, if this can be squeezed into 0.13, I'm willing to rush to get it done.

hmm though has done this oh well

it's easy just change this routine

coerce_scalar_to_timedelta_type(

in tseries/time delta

just need a isnull check and return tslib.iNaT
(it will work now if u pass NaT)


Resolving patch:
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -1162,15 +1162,25 @@ def _try_fill(self, value):
 
     def _try_coerce_args(self, values, other):
         """ provide coercion to our input arguments
-            we are going to compare vs i8, so coerce to integer
-            values is always ndarra like, other may not be """
-        values = values.view('i8')
+            we are going to compare vs i8, so coerce to floats
+            repring NaT with np.nan so nans propagate
+            values is always ndarray like, other may not be """
+        def masker(v):
+            mask = isnull(v)
+            v = v.view('i8').astype('float64')
+            v[mask] = np.nan
+            return v
+
+        values = masker(values)
+
         if isnull(other) or (np.isscalar(other) and other == tslib.iNaT):
-            other = tslib.iNaT
+            other = np.nan
         elif isinstance(other, np.timedelta64):
             other = _coerce_scalar_to_timedelta_type(other,unit='s').item()
+            if other == tslib.iNaT:
+                other = np.nan
         else:
-            other = other.view('i8')
+            other = masker(other)
 
         return values, other
 
diff --git a/pandas/core/ops.py b/pandas/core/ops.py
--- a/pandas/core/ops.py
+++ b/pandas/core/ops.py
@@ -255,7 +255,7 @@ def __init__(self, left, right, name):
         self.name = name
 
         lvalues = self._convert_to_array(left, name=name)
-        rvalues = self._convert_to_array(right, name=name)
+        rvalues = self._convert_to_array(right, name=name, other=lvalues)
 
         self.is_timedelta_lhs = com.is_timedelta64_dtype(left)
         self.is_datetime_lhs = com.is_datetime64_dtype(left)
@@ -317,7 +317,7 @@ def _validate(self):
                             'of a series/ndarray of type datetime64[ns] '
                             'or a timedelta')
 
-    def _convert_to_array(self, values, name=None):
+    def _convert_to_array(self, values, name=None, other=None):
         """converts values to ndarray"""
         from pandas.tseries.timedeltas import _possibly_cast_to_timedelta
 
@@ -325,9 +325,16 @@ def _convert_to_array(self, values, name=None):
         if not is_list_like(values):
             values = np.array([values])
         inferred_type = lib.infer_dtype(values)
+
         if inferred_type in ('datetime64', 'datetime', 'date', 'time'):
+            # if we have a other of timedelta, but use pd.NaT here we
+            # we are in the wrong path
+            if other is not None and other.dtype == 'timedelta64[ns]' and all(isnull(v) for v in values):
+                values = np.empty(values.shape,dtype=other.dtype)
+                values[:] = tslib.iNaT
+
             # a datetlike
-            if not (isinstance(values, (pa.Array, pd.Series)) and
+            elif not (isinstance(values, (pa.Array, pd.Series)) and
                     com.is_datetime64_dtype(values)):
                 values = tslib.array_to_datetime(values)
             elif isinstance(values, pd.DatetimeIndex):
@@ -354,6 +361,15 @@ def _convert_to_array(self, values, name=None):
                                     ', '.join([com.pprint_thing(v)
                                                for v in values[mask]])))
             values = _possibly_cast_to_timedelta(os, coerce=coerce)
+        elif inferred_type == 'floating':
+
+            # all nan, so ok, use the other dtype (e.g. timedelta or datetime)
+            if isnull(values).all():
+                values = np.empty(values.shape,dtype=other.dtype)
+                values[:] = tslib.iNaT
+            else:
+                raise TypeError("incompatible type [{0}] for a datetime/timedelta"
+                                " operation".format(pa.array(values).dtype))
         else:
             raise TypeError("incompatible type [{0}] for a datetime/timedelta"
                             " operation".format(pa.array(values).dtype))
@@ -452,6 +468,8 @@ def na_op(x, y):
 
     def wrapper(left, right, name=name):
 
+        if isinstance(right, pd.DataFrame):
+            return NotImplemented
         time_converted = _TimeOp.maybe_convert_for_time_op(left, right, name)
 
         if time_converted is None:
@@ -488,8 +506,6 @@ def wrapper(left, right, name=name):
 
             return left._constructor(wrap_results(arr), index=index,
                                      name=name, dtype=dtype)
-        elif isinstance(right, pd.DataFrame):
-            return NotImplemented
         else:
             # scalars
             if hasattr(lvalues, 'values'):
diff --git a/pandas/tseries/timedeltas.py b/pandas/tseries/timedeltas.py
--- a/pandas/tseries/timedeltas.py
+++ b/pandas/tseries/timedeltas.py
@@ -9,7 +9,7 @@
 import pandas.tslib as tslib
 from pandas import compat, _np_version_under1p7
 from pandas.core.common import (ABCSeries, is_integer, is_timedelta64_dtype,
-                                _values_from_object, is_list_like)
+                                _values_from_object, is_list_like, isnull)
 
 repr_timedelta = tslib.repr_timedelta64
 repr_timedelta64 = tslib.repr_timedelta64
@@ -84,6 +84,8 @@ def conv(v):
         r = conv(r)
     elif r == tslib.iNaT:
         return r
+    elif isnull(r):
+        return np.timedelta64('NaT')
     elif isinstance(r, np.timedelta64):
         r = r.astype("m8[{0}]".format(unit.lower()))
     elif is_integer(r):


Test patch:
diff --git a/pandas/tseries/tests/test_timedeltas.py b/pandas/tseries/tests/test_timedeltas.py
--- a/pandas/tseries/tests/test_timedeltas.py
+++ b/pandas/tseries/tests/test_timedeltas.py
@@ -195,6 +195,122 @@ def test_timedelta_ops(self):
         expected = to_timedelta('00:00:08')
         tm.assert_almost_equal(result, expected)
 
+    def test_to_timedelta_on_missing_values(self):
+        _skip_if_numpy_not_friendly()
+
+        # GH5438
+        timedelta_NaT = np.timedelta64('NaT')
+
+        actual = pd.to_timedelta(Series(['00:00:01', np.nan]))
+        expected = Series([np.timedelta64(1000000000, 'ns'), timedelta_NaT], dtype='<m8[ns]')
+        assert_series_equal(actual, expected)
+
+        actual = pd.to_timedelta(Series(['00:00:01', pd.NaT]))
+        assert_series_equal(actual, expected)
+
+        actual = pd.to_timedelta(np.nan)
+        self.assert_(actual == timedelta_NaT)
+
+        actual = pd.to_timedelta(pd.NaT)
+        self.assert_(actual == timedelta_NaT)
+
+    def test_timedelta_ops_with_missing_values(self):
+        _skip_if_numpy_not_friendly()
+
+        # setup
+        s1 = pd.to_timedelta(Series(['00:00:01']))
+        s2 = pd.to_timedelta(Series(['00:00:02']))
+        sn = pd.to_timedelta(Series([pd.NaT]))
+        df1 = DataFrame(['00:00:01']).apply(pd.to_timedelta)
+        df2 = DataFrame(['00:00:02']).apply(pd.to_timedelta)
+        dfn = DataFrame([pd.NaT]).apply(pd.to_timedelta)
+        scalar1 = pd.to_timedelta('00:00:01')
+        scalar2 = pd.to_timedelta('00:00:02')
+        timedelta_NaT = pd.to_timedelta('NaT')
+        NA = np.nan
+
+        actual = scalar1 + scalar1
+        self.assert_(actual == scalar2)
+        actual = scalar2 - scalar1
+        self.assert_(actual == scalar1)
+
+        actual = s1 + s1
+        assert_series_equal(actual, s2)
+        actual = s2 - s1
+        assert_series_equal(actual, s1)
+
+        actual = s1 + scalar1
+        assert_series_equal(actual, s2)
+        actual = s2 - scalar1
+        assert_series_equal(actual, s1)
+
+        actual = s1 + timedelta_NaT
+        assert_series_equal(actual, sn)
+        actual = s1 - timedelta_NaT
+        assert_series_equal(actual, sn)
+
+        actual = s1 + NA
+        assert_series_equal(actual, sn)
+        actual = s1 - NA
+        assert_series_equal(actual, sn)
+
+        actual = s1 + pd.NaT  # NaT is datetime, not timedelta
+        assert_series_equal(actual, sn)
+        actual = s2 - pd.NaT
+        assert_series_equal(actual, sn)
+
+        actual = s1 + df1
+        assert_frame_equal(actual, df2)
+        actual = s2 - df1
+        assert_frame_equal(actual, df1)
+        actual = df1 + s1
+        assert_frame_equal(actual, df2)
+        actual = df2 - s1
+        assert_frame_equal(actual, df1)
+
+        actual = df1 + df1
+        assert_frame_equal(actual, df2)
+        actual = df2 - df1
+        assert_frame_equal(actual, df1)
+
+        actual = df1 + scalar1
+        assert_frame_equal(actual, df2)
+        actual = df2 - scalar1
+        assert_frame_equal(actual, df1)
+
+        actual = df1 + timedelta_NaT
+        assert_frame_equal(actual, dfn)
+        actual = df1 - timedelta_NaT
+        assert_frame_equal(actual, dfn)
+
+        actual = df1 + NA
+        assert_frame_equal(actual, dfn)
+        actual = df1 - NA
+        assert_frame_equal(actual, dfn)
+
+        actual = df1 + pd.NaT  # NaT is datetime, not timedelta
+        assert_frame_equal(actual, dfn)
+        actual = df1 - pd.NaT
+        assert_frame_equal(actual, dfn)
+
+    def test_apply_to_timedelta(self):
+        _skip_if_numpy_not_friendly()
+
+        timedelta_NaT = pd.to_timedelta('NaT')
+
+        list_of_valid_strings = ['00:00:01', '00:00:02']
+        a = pd.to_timedelta(list_of_valid_strings)
+        b = Series(list_of_valid_strings).apply(pd.to_timedelta)
+        # Can't compare until apply on a Series gives the correct dtype
+        # assert_series_equal(a, b)
+
+        list_of_strings = ['00:00:01', np.nan, pd.NaT, timedelta_NaT]
+        a = pd.to_timedelta(list_of_strings)
+        b = Series(list_of_strings).apply(pd.to_timedelta)
+        # Can't compare until apply on a Series gives the correct dtype
+        # assert_series_equal(a, b)
+
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)