Repository: pandas-dev/pandas

Issue:
fillna() on Series or DataFrame containing datetime64 mess the values
s = pd.Series([pd.NaT, pd.NaT, '2013-08-05 15:30:00.**000001**'])
print s
0                          NaT
1                          NaT
2   2013-08-05 15:30:00.**000001**
dtype: datetime64[ns]

print s.fillna(method='backfill')
0   2013-08-05 15:30:00.**000001024**
1   2013-08-05 15:30:00.**000001024**
2   2013-08-05 15:30:00.**000001024**
dtype: datetime64[ns]

This happens with pandas 0.13.1 and numpy 1.8.0



Discussion:
hmm...looks like a rounding/precision issue.

care to have a look into this?

its sending this to the method with the dtype stripped, so it ends up converting to float and back (rather than using the correct datetime routines)....


Resolving patch:
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -1244,13 +1244,14 @@ def wrapper(arr, mask, limit=None):
                                         np.int64)
 
 
-def pad_1d(values, limit=None, mask=None):
+def pad_1d(values, limit=None, mask=None, dtype=None):
 
-    dtype = values.dtype.name
+    if dtype is None:
+        dtype = values.dtype
     _method = None
     if is_float_dtype(values):
-        _method = getattr(algos, 'pad_inplace_%s' % dtype, None)
-    elif is_datetime64_dtype(values):
+        _method = getattr(algos, 'pad_inplace_%s' % dtype.name, None)
+    elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values):
         _method = _pad_1d_datetime
     elif is_integer_dtype(values):
         values = _ensure_float64(values)
@@ -1259,7 +1260,7 @@ def pad_1d(values, limit=None, mask=None):
         _method = algos.pad_inplace_object
 
     if _method is None:
-        raise ValueError('Invalid dtype for pad_1d [%s]' % dtype)
+        raise ValueError('Invalid dtype for pad_1d [%s]' % dtype.name)
 
     if mask is None:
         mask = isnull(values)
@@ -1268,13 +1269,14 @@ def pad_1d(values, limit=None, mask=None):
     return values
 
 
-def backfill_1d(values, limit=None, mask=None):
+def backfill_1d(values, limit=None, mask=None, dtype=None):
 
-    dtype = values.dtype.name
+    if dtype is None:
+        dtype = values.dtype
     _method = None
     if is_float_dtype(values):
-        _method = getattr(algos, 'backfill_inplace_%s' % dtype, None)
-    elif is_datetime64_dtype(values):
+        _method = getattr(algos, 'backfill_inplace_%s' % dtype.name, None)
+    elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values):
         _method = _backfill_1d_datetime
     elif is_integer_dtype(values):
         values = _ensure_float64(values)
@@ -1283,7 +1285,7 @@ def backfill_1d(values, limit=None, mask=None):
         _method = algos.backfill_inplace_object
 
     if _method is None:
-        raise ValueError('Invalid dtype for backfill_1d [%s]' % dtype)
+        raise ValueError('Invalid dtype for backfill_1d [%s]' % dtype.name)
 
     if mask is None:
         mask = isnull(values)
@@ -1293,13 +1295,14 @@ def backfill_1d(values, limit=None, mask=None):
     return values
 
 
-def pad_2d(values, limit=None, mask=None):
+def pad_2d(values, limit=None, mask=None, dtype=None):
 
-    dtype = values.dtype.name
+    if dtype is None:
+        dtype = values.dtype
     _method = None
     if is_float_dtype(values):
-        _method = getattr(algos, 'pad_2d_inplace_%s' % dtype, None)
-    elif is_datetime64_dtype(values):
+        _method = getattr(algos, 'pad_2d_inplace_%s' % dtype.name, None)
+    elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values):
         _method = _pad_2d_datetime
     elif is_integer_dtype(values):
         values = _ensure_float64(values)
@@ -1308,7 +1311,7 @@ def pad_2d(values, limit=None, mask=None):
         _method = algos.pad_2d_inplace_object
 
     if _method is None:
-        raise ValueError('Invalid dtype for pad_2d [%s]' % dtype)
+        raise ValueError('Invalid dtype for pad_2d [%s]' % dtype.name)
 
     if mask is None:
         mask = isnull(values)
@@ -1322,13 +1325,14 @@ def pad_2d(values, limit=None, mask=None):
     return values
 
 
-def backfill_2d(values, limit=None, mask=None):
+def backfill_2d(values, limit=None, mask=None, dtype=None):
 
-    dtype = values.dtype.name
+    if dtype is None:
+        dtype = values.dtype
     _method = None
     if is_float_dtype(values):
-        _method = getattr(algos, 'backfill_2d_inplace_%s' % dtype, None)
-    elif is_datetime64_dtype(values):
+        _method = getattr(algos, 'backfill_2d_inplace_%s' % dtype.name, None)
+    elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values):
         _method = _backfill_2d_datetime
     elif is_integer_dtype(values):
         values = _ensure_float64(values)
@@ -1337,7 +1341,7 @@ def backfill_2d(values, limit=None, mask=None):
         _method = algos.backfill_2d_inplace_object
 
     if _method is None:
-        raise ValueError('Invalid dtype for backfill_2d [%s]' % dtype)
+        raise ValueError('Invalid dtype for backfill_2d [%s]' % dtype.name)
 
     if mask is None:
         mask = isnull(values)
@@ -1503,7 +1507,7 @@ def _interpolate_scipy_wrapper(x, y, new_x, method, fill_value=None,
     return new_y
 
 
-def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None):
+def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None, dtype=None):
     """ perform an actual interpolation of values, values will be make 2-d if
     needed fills inplace, returns the result
     """
@@ -1525,9 +1529,9 @@ def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None):
 
     method = _clean_fill_method(method)
     if method == 'pad':
-        values = transf(pad_2d(transf(values), limit=limit, mask=mask))
+        values = transf(pad_2d(transf(values), limit=limit, mask=mask, dtype=dtype))
     else:
-        values = transf(backfill_2d(transf(values), limit=limit, mask=mask))
+        values = transf(backfill_2d(transf(values), limit=limit, mask=mask, dtype=dtype))
 
     # reshape back
     if ndim == 1:
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -878,7 +878,12 @@ def _interpolate_with_fill(self, method='pad', axis=0, inplace=False,
         fill_value = self._try_fill(fill_value)
         values = self.values if inplace else self.values.copy()
         values = self._try_operate(values)
-        values = com.interpolate_2d(values, method, axis, limit, fill_value)
+        values = com.interpolate_2d(values,
+                                    method=method,
+                                    axis=axis,
+                                    limit=limit,
+                                    fill_value=fill_value,
+                                    dtype=self.dtype)
         values = self._try_coerce_result(values)
 
         blocks = [make_block(values, self.items, self.ref_items,


Test patch:
diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py
--- a/pandas/tests/test_series.py
+++ b/pandas/tests/test_series.py
@@ -2854,6 +2854,14 @@ def test_datetime64_fillna(self):
                            Timestamp('20130103 9:01:01')])
         assert_series_equal(result, expected)
 
+        # GH 6587
+        # make sure that we are treating as integer when filling
+        s = Series([pd.NaT, pd.NaT, '2013-08-05 15:30:00.000001'])
+        expected = Series(['2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001'], dtype='M8[ns]')
+        result = s.fillna(method='backfill')
+        assert_series_equal(result, expected)
+
+
     def test_fillna_int(self):
         s = Series(np.random.randint(-100, 100, 50))
         s.fillna(method='ffill', inplace=True)