Repository: pandas-dev/pandas

Issue:
Assigning timedelta64 to new column casts to float instead of filling missing values with NaT
When assigning a `timedelta64` array to a subset of a new column of a `DataFrame`, missing data is not filled with `NaT` as expected; rather, the new column is cast to `float64` and `NaN` is used instead.  This cast does not usually occur when all values are present, _except_ when there are already `float64` columns but no `timedelta64` columns in the `DataFrame` _and_ indexing is done through `.ix` or `.loc`.

It's possible these should be two separate issues.

There are a lot of issues involving `NaT` in the issue tracker; I'm not 100% sure that this isn't a duplicate.  (Nor am I 100% sure this isn't intended behavior, but if it is I'd expect it to be documented more prominently.)

``` python

import numpy as np
import pandas as pd

one_hour = 60*60*10**9

temp = pd.DataFrame({}, index=pd.date_range('2014-1-1', periods=4))
temp['A'] = np.array([1*one_hour]*4, dtype='m8[ns]')
temp.loc[:,'B'] = np.array([2*one_hour]*4, dtype='m8[ns]')
temp.loc[:3,'C'] = np.array([3*one_hour]*3, dtype='m8[ns]')
temp.ix[:,'D'] = np.array([4*one_hour]*4, dtype='m8[ns]')
temp.ix[:3,'E'] = np.array([5*one_hour]*3, dtype='m8[ns]')
temp['F'] = np.timedelta64('NaT')
temp.ix[:-1,'F'] = np.array([6*one_hour]*3, dtype='m8[ns]')

temp
#                   A        B             C        D             E        F
#2014-01-01 01:00:00 02:00:00  1.080000e+13 04:00:00  1.800000e+13 06:00:00
#2014-01-02 01:00:00 02:00:00  1.080000e+13 04:00:00  1.800000e+13 06:00:00
#2014-01-03 01:00:00 02:00:00  1.080000e+13 04:00:00  1.800000e+13 06:00:00
#2014-01-04 01:00:00 02:00:00           NaN 04:00:00           NaN      NaT
# 
# [4 rows x 6 columns]

temp = pd.DataFrame({}, index=pd.date_range('2014-1-1', periods=4))
# Partial assignment converts
temp.ix[:-1,'A'] = np.array([1*one_hour]*3, dtype='m8[ns]')
# DataFrame is all floats; converts
temp.ix[:,'B'] = np.array([2*one_hour]*4, dtype='m8[ns]')
# .ix and .loc behave the same
temp.loc[:,'C'] = np.array([3*one_hour]*4, dtype='m8[ns]')
# straight column assignment doesn't convert
temp['D'] = np.array([4*one_hour]*4, dtype='m8[ns]')
# Now there are timedeltas; doesn't convert
temp.ix[:,'E'] = np.array([5*one_hour]*4, dtype='m8[ns]')
# .ix and .loc still behave the same
temp.loc[:,'F'] = np.array([6*one_hour]*4, dtype='m8[ns]')

temp
#                        A             B             C        D        E  \
#2014-01-01  3.600000e+12  7.200000e+12  1.080000e+13 04:00:00 05:00:00   
#2014-01-02  3.600000e+12  7.200000e+12  1.080000e+13 04:00:00 05:00:00   
#2014-01-03  3.600000e+12  7.200000e+12  1.080000e+13 04:00:00 05:00:00   
#2014-01-04           NaN  7.200000e+12  1.080000e+13 04:00:00 05:00:00   
# 
#                   F  
#2014-01-01 06:00:00  
#2014-01-02 06:00:00  
#2014-01-03 06:00:00  
#2014-01-04 06:00:00  
# 
# [4 rows x 6 columns]

temp = pd.DataFrame({}, index=pd.date_range('2014-1-1', periods=4))
# No columns yet, no conversion
temp.ix[:,'A'] = np.array([2*one_hour]*4, dtype='m8[ns]')
#                   A
#2014-01-01 02:00:00
#2014-01-02 02:00:00
#2014-01-03 02:00:00
#2014-01-04 02:00:00
# 
# [4 rows x 1 columns]

```



Discussion:
show your numpy / pandas versions

Oops.

``` python

pd.show_versions()
# 
# INSTALLED VERSIONS
# ------------------
# commit: None
# python: 2.7.5.final.0
# python-bits: 64
# OS: Darwin
# OS-release: 13.2.0
# machine: x86_64
# processor: i386
# byteorder: little
# LC_ALL: None
# LANG: en_US.UTF-8
# 
# pandas: 0.13.1
# Cython: None
# numpy: 1.8.0
# scipy: 0.12.1
# statsmodels: None
# IPython: 1.1.0
# sphinx: None
# patsy: None
# scikits.timeseries: None
# dateutil: 2.2
# pytz: 2013.9
# bottleneck: None
# tables: None
# numexpr: None
# matplotlib: 1.3.0
# openpyxl: None
# xlrd: None
# xlwt: None
# xlsxwriter: None
# sqlalchemy: None
# lxml: None
# bs4: None
# html5lib: None
# bq: None
# apiclient: None
```

hmm, this is a partial assignment with dtype inference. Looks like a bug (e.g. this works for a rhs of `datestime64[ns]`). Welcome to have you dig in!


Resolving patch:
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -2603,7 +2603,7 @@ def check_main():
 def in_qtconsole():
     """
     check if we're inside an IPython qtconsole
-    
+
     DEPRECATED: This is no longer needed, or working, in IPython 3 and above.
     """
     try:
@@ -2622,7 +2622,7 @@ def in_qtconsole():
 def in_ipnb():
     """
     check if we're inside an IPython Notebook
-    
+
     DEPRECATED: This is no longer used in pandas, and won't work in IPython 3
     and above.
     """
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -1037,9 +1037,11 @@ class FloatBlock(FloatOrComplexBlock):
     def _can_hold_element(self, element):
         if is_list_like(element):
             element = np.array(element)
-            return issubclass(element.dtype.type, (np.floating, np.integer))
-        return (isinstance(element, (float, int, np.float_, np.int_)) and
-                not isinstance(bool, np.bool_))
+            tipo = element.dtype.type
+            return issubclass(tipo, (np.floating, np.integer)) and not issubclass(
+                tipo, (np.datetime64, np.timedelta64))
+        return isinstance(element, (float, int, np.float_, np.int_)) and not isinstance(
+            element, (bool, np.bool_, datetime, timedelta, np.datetime64, np.timedelta64))
 
     def _try_cast(self, element):
         try:
@@ -1099,7 +1101,8 @@ class IntBlock(NumericBlock):
     def _can_hold_element(self, element):
         if is_list_like(element):
             element = np.array(element)
-            return issubclass(element.dtype.type, np.integer)
+            tipo = element.dtype.type
+            return issubclass(tipo, np.integer) and not issubclass(tipo, (np.datetime64, np.timedelta64))
         return com.is_integer(element)
 
     def _try_cast(self, element):


Test patch:
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -32,7 +32,8 @@
 import pandas.core.format as fmt
 import pandas.core.datetools as datetools
 from pandas import (DataFrame, Index, Series, notnull, isnull,
-                    MultiIndex, DatetimeIndex, Timestamp, date_range, read_csv)
+                    MultiIndex, DatetimeIndex, Timestamp, date_range, read_csv,
+                    _np_version_under1p7)
 import pandas as pd
 from pandas.parser import CParserError
 from pandas.util.misc import is_little_endian
@@ -2180,11 +2181,11 @@ def test_set_index_cast_datetimeindex(self):
         # reset_index with single level
         for tz in ['UTC', 'Asia/Tokyo', 'US/Eastern']:
             idx = pd.date_range('1/1/2011', periods=5, freq='D', tz=tz, name='idx')
-            df = pd.DataFrame({'a': range(5), 'b': ['A', 'B', 'C', 'D', 'E']}, index=idx) 
+            df = pd.DataFrame({'a': range(5), 'b': ['A', 'B', 'C', 'D', 'E']}, index=idx)
 
             expected = pd.DataFrame({'idx': [datetime(2011, 1, 1), datetime(2011, 1, 2),
                                              datetime(2011, 1, 3), datetime(2011, 1, 4),
-                                             datetime(2011, 1, 5)], 
+                                             datetime(2011, 1, 5)],
                                      'a': range(5), 'b': ['A', 'B', 'C', 'D', 'E']},
                                      columns=['idx', 'a', 'b'])
             expected['idx'] = expected['idx'].apply(lambda d: pd.Timestamp(d, tz=tz))
@@ -3757,6 +3758,28 @@ def test_operators_timedelta64(self):
         self.assertTrue(df['off1'].dtype == 'timedelta64[ns]')
         self.assertTrue(df['off2'].dtype == 'timedelta64[ns]')
 
+    def test_datetimelike_setitem_with_inference(self):
+        if _np_version_under1p7:
+            raise nose.SkipTest("numpy < 1.7")
+
+        # GH 7592
+        # assignment of timedeltas with NaT
+
+        one_hour = timedelta(hours=1)
+        df = DataFrame(index=date_range('20130101',periods=4))
+        df['A'] = np.array([1*one_hour]*4, dtype='m8[ns]')
+        df.loc[:,'B'] = np.array([2*one_hour]*4, dtype='m8[ns]')
+        df.loc[:3,'C'] = np.array([3*one_hour]*3, dtype='m8[ns]')
+        df.ix[:,'D'] = np.array([4*one_hour]*4, dtype='m8[ns]')
+        df.ix[:3,'E'] = np.array([5*one_hour]*3, dtype='m8[ns]')
+        df['F'] = np.timedelta64('NaT')
+        df.ix[:-1,'F'] = np.array([6*one_hour]*3, dtype='m8[ns]')
+        df.ix[-3:,'G'] = date_range('20130101',periods=3)
+        df['H'] = np.datetime64('NaT')
+        result = df.dtypes
+        expected = Series([np.dtype('timedelta64[ns]')]*6+[np.dtype('datetime64[ns]')]*2,index=list('ABCDEFGH'))
+        assert_series_equal(result,expected)
+
     def test_new_empty_index(self):
         df1 = DataFrame(randn(0, 3))
         df2 = DataFrame(randn(0, 3))