Repository: pandas-dev/pandas

Issue:
series created with DatetimeIndex when index is [None]
This looks like a change from 0.13.1 to HEAD. If you create a series like this:

``` python
pd.Series(index=np.array([None]))
```

Then in 0.13.1 the result is:

``` python
Out[16]: 
NaN   NaN
dtype: float64

In [19]: x.index.dtype
Out[19]: dtype('O')

In [20]: type(x.index)
Out[20]: pandas.core.index.Index
```

but in HEAD the result is 

``` python
Out[191]: 
NaT   NaN
dtype: float64

In[192]: x.index.dtype
Out[192]: dtype('<M8[ns]')

In[193]: type(x.index)
Out[193]: pandas.tseries.index.DatetimeIndex
```

So the index has changed from Index/object to DatetimeIndex/datetime64.



Discussion:
Running through the construction of a pd.Series object in the debugger, it looks like this is happening in  `Series._set_axis` because `is_all_dates` is `True` for an index with just `None` in it.

``` python
    def _set_axis(self, axis, labels, fastpath=False):
        """ override generic, we want to set the _typ here """

        if not fastpath:
            labels = _ensure_index(labels)

        is_all_dates = labels.is_all_dates
        if is_all_dates:
            from pandas.tseries.index import DatetimeIndex
            from pandas.tseries.period import PeriodIndex
            if not isinstance(labels, (DatetimeIndex, PeriodIndex)):
                labels = DatetimeIndex(labels)

        .....
```

Constructing an `Index` manually confirms this:

``` python
i = pd.Index(np.array([None]))
i.is_all_dates    
Out[208]: True
```

But `is_all_dates` just defers to `is_datetime_array` which I think is `cimport`ed from `numpy`

Looks like it's [PR 6341](https://github.com/pydata/pandas/pull/6341) which made the change: 

no `is_datetime_array` is defined in `src/inference.pyx`; a single element `None` in this context should be simply `nan`. hmm. 

cc @immerrr


Resolving patch:
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -1782,24 +1782,81 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
             value.dtype == np.object_)):
             pass
 
+        # try to infer if we have a datetimelike here
+        # otherwise pass thru
         else:
-            # we might have a array (or single object) that is datetime like,
-            # and no dtype is passed don't change the value unless we find a
-            # datetime set
-            v = value
-            if not is_list_like(v):
-                v = [v]
-            if len(v):
-                inferred_type = lib.infer_dtype(v)
-                if inferred_type in ['datetime', 'datetime64']:
-                    try:
-                        value = tslib.array_to_datetime(np.array(v))
-                    except:
-                        pass
-                elif inferred_type in ['timedelta', 'timedelta64']:
-                    from pandas.tseries.timedeltas import \
-                        _possibly_cast_to_timedelta
-                    value = _possibly_cast_to_timedelta(value, coerce='compat')
+            value = _possibly_infer_to_datetimelike(value)
+
+    return value
+
+def _possibly_infer_to_datetimelike(value):
+    # we might have a array (or single object) that is datetime like,
+    # and no dtype is passed don't change the value unless we find a
+    # datetime/timedelta set
+
+    # this is pretty strict in that a datetime/timedelta is REQUIRED
+    # in addition to possible nulls/string likes
+
+    # ONLY strings are NOT datetimelike
+
+    v = value
+    if not is_list_like(v):
+        v = [v]
+    if not isinstance(v, np.ndarray):
+        v = np.array(v)
+    shape = v.shape
+    if not v.ndim == 1:
+        v = v.ravel()
+
+    if len(v):
+
+        def _try_datetime(v):
+            # safe coerce to datetime64
+            try:
+                return tslib.array_to_datetime(v, raise_=True).reshape(shape)
+            except:
+                return v
+
+        def _try_timedelta(v):
+            # safe coerce to timedelta64
+
+            # will try first with a string & object conversion
+            from pandas.tseries.timedeltas import to_timedelta
+            try:
+                return to_timedelta(v).values.reshape(shape)
+            except:
+
+                # this is for compat with numpy < 1.7
+                # but string-likes will fail here
+
+                from pandas.tseries.timedeltas import \
+                     _possibly_cast_to_timedelta
+                try:
+                    return _possibly_cast_to_timedelta(v, coerce='compat').reshape(shape)
+                except:
+                    return v
+
+        # do a quick inference for perf
+        sample = v[:min(3,len(v))]
+        inferred_type = lib.infer_dtype(sample)
+
+        if inferred_type in ['datetime', 'datetime64']:
+            value = _try_datetime(v)
+        elif inferred_type in ['timedelta', 'timedelta64']:
+            value = _try_timedelta(v)
+
+        # its possible to have nulls intermixed within the datetime or timedelta
+        # these will in general have an inferred_type of 'mixed', so have to try
+        # both datetime and timedelta
+
+        # try timedelta first to avoid spurious datetime conversions
+        # e.g. '00:00:01' is a timedelta but technically is also a datetime
+        elif inferred_type in ['mixed']:
+
+            if lib.is_possible_datetimelike_array(_ensure_object(v)):
+                value = _try_timedelta(v)
+                if lib.infer_dtype(value) in ['mixed']:
+                    value = _try_datetime(v)
 
     return value
 
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -12,7 +12,8 @@
                                 _NS_DTYPE, _TD_DTYPE, ABCSeries, is_list_like,
                                 ABCSparseSeries, _infer_dtype_from_scalar,
                                 _is_null_datelike_scalar,
-                                is_timedelta64_dtype, is_datetime64_dtype,)
+                                is_timedelta64_dtype, is_datetime64_dtype,
+                                _possibly_infer_to_datetimelike)
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.core.indexing import (_maybe_convert_indices, _length_of_indexer)
 import pandas.core.common as com
@@ -1807,26 +1808,21 @@ def make_block(values, placement, klass=None, ndim=None,
         elif issubclass(vtype, np.complexfloating):
             klass = ComplexBlock
 
-        # try to infer a DatetimeBlock, or set to an ObjectBlock
         else:
 
+            # we want to infer here if its a datetimelike if its object type
+            # this is pretty strict in that it requires a datetime/timedelta
+            # value IN addition to possible nulls/strings
+            # an array of ONLY strings will not be inferred
             if np.prod(values.shape):
-                flat = values.ravel()
-
-                # try with just the first element; we just need to see if
-                # this is a datetime or not
-                inferred_type = lib.infer_dtype(flat[0:1])
-                if inferred_type in ['datetime', 'datetime64']:
-
-                    # we have an object array that has been inferred as
-                    # datetime, so convert it
-                    try:
-                        values = tslib.array_to_datetime(
-                            flat).reshape(values.shape)
-                        if issubclass(values.dtype.type, np.datetime64):
-                            klass = DatetimeBlock
-                    except:  # it already object, so leave it
-                        pass
+                result = _possibly_infer_to_datetimelike(values)
+                vtype = result.dtype.type
+                if issubclass(vtype, np.datetime64):
+                    klass = DatetimeBlock
+                    values = result
+                elif (issubclass(vtype, np.timedelta64)):
+                    klass = TimeDeltaBlock
+                    values = result
 
             if klass is None:
                 klass = ObjectBlock
@@ -2525,7 +2521,7 @@ def _consolidate_inplace(self):
             self._known_consolidated = True
             self._rebuild_blknos_and_blklocs()
 
-    def get(self, item):
+    def get(self, item, fastpath=True):
         """
         Return values for selected item (ndarray or BlockManager).
         """
@@ -2543,7 +2539,7 @@ def get(self, item):
                     else:
                         raise ValueError("cannot label index with a null key")
 
-            return self.iget(loc)
+            return self.iget(loc, fastpath=fastpath)
         else:
 
             if isnull(item):
@@ -2553,8 +2549,25 @@ def get(self, item):
             return self.reindex_indexer(new_axis=self.items[indexer],
                                         indexer=indexer, axis=0, allow_dups=True)
 
-    def iget(self, i):
-        return self.blocks[self._blknos[i]].iget(self._blklocs[i])
+    def iget(self, i, fastpath=True):
+        """
+        Return the data as a SingleBlockManager if fastpath=True and possible
+
+        Otherwise return as a ndarray
+
+        """
+
+        block = self.blocks[self._blknos[i]]
+        values = block.iget(self._blklocs[i])
+        if not fastpath or block.is_sparse or values.ndim != 1:
+            return values
+
+        # fastpath shortcut for select a single-dim from a 2-dim BM
+        return SingleBlockManager([ block.make_block_same_class(values,
+                                                                placement=slice(0, len(values)),
+                                                                fastpath=True) ],
+                                  self.axes[1])
+
 
     def get_scalar(self, tup):
         """
diff --git a/pandas/tseries/timedeltas.py b/pandas/tseries/timedeltas.py
--- a/pandas/tseries/timedeltas.py
+++ b/pandas/tseries/timedeltas.py
@@ -9,7 +9,7 @@
 import pandas.tslib as tslib
 from pandas import compat, _np_version_under1p7
 from pandas.core.common import (ABCSeries, is_integer, is_integer_dtype, is_timedelta64_dtype,
-                                _values_from_object, is_list_like, isnull)
+                                _values_from_object, is_list_like, isnull, _ensure_object)
 
 repr_timedelta = tslib.repr_timedelta64
 repr_timedelta64 = tslib.repr_timedelta64
@@ -46,7 +46,7 @@ def _convert_listlike(arg, box, unit):
             value = arg.astype('timedelta64[{0}]'.format(unit)).astype('timedelta64[ns]')
         else:
             try:
-                value = tslib.array_to_timedelta64(_ensure_object(arg),unit=unit)
+                value = tslib.array_to_timedelta64(_ensure_object(arg), unit=unit)
             except:
                 value = np.array([ _coerce_scalar_to_timedelta_type(r, unit=unit) for r in arg ])
 


Test patch:
diff --git a/pandas/io/tests/test_json/test_pandas.py b/pandas/io/tests/test_json/test_pandas.py
--- a/pandas/io/tests/test_json/test_pandas.py
+++ b/pandas/io/tests/test_json/test_pandas.py
@@ -4,8 +4,8 @@
 import os
 
 import numpy as np
-
-from pandas import Series, DataFrame, DatetimeIndex, Timestamp
+import nose
+from pandas import Series, DataFrame, DatetimeIndex, Timestamp, _np_version_under1p7
 import pandas as pd
 read_json = pd.read_json
 
@@ -600,11 +600,29 @@ def test_url(self):
         for c in ['created_at', 'closed_at', 'updated_at']:
             self.assertEqual(result[c].dtype, 'datetime64[ns]')
 
-    def test_default_handler(self):
+    def test_timedelta(self):
+        if _np_version_under1p7:
+            raise nose.SkipTest("numpy < 1.7")
+
         from datetime import timedelta
+        converter = lambda x: pd.to_timedelta(x,unit='ms')
+
+        s = Series([timedelta(23), timedelta(seconds=5)])
+        self.assertEqual(s.dtype,'timedelta64[ns]')
+        assert_series_equal(s, pd.read_json(s.to_json(),typ='series').apply(converter))
+
         frame = DataFrame([timedelta(23), timedelta(seconds=5)])
+        self.assertEqual(frame[0].dtype,'timedelta64[ns]')
+        assert_frame_equal(
+            frame, pd.read_json(frame.to_json()).apply(converter))
+
+    def test_default_handler(self):
+        from datetime import timedelta
+
+        frame = DataFrame([timedelta(23), timedelta(seconds=5), 42])
         self.assertRaises(OverflowError, frame.to_json)
-        expected = DataFrame([str(timedelta(23)), str(timedelta(seconds=5))])
+
+        expected = DataFrame([str(timedelta(23)), str(timedelta(seconds=5)), 42])
         assert_frame_equal(
             expected, pd.read_json(frame.to_json(default_handler=str)))
 
diff --git a/pandas/tests/test_internals.py b/pandas/tests/test_internals.py
--- a/pandas/tests/test_internals.py
+++ b/pandas/tests/test_internals.py
@@ -356,7 +356,9 @@ def test_get_scalar(self):
         for item in self.mgr.items:
             for i, index in enumerate(self.mgr.axes[1]):
                 res = self.mgr.get_scalar((item, index))
-                exp = self.mgr.get(item)[i]
+                exp = self.mgr.get(item, fastpath=False)[i]
+                assert_almost_equal(res, exp)
+                exp = self.mgr.get(item).values[i]
                 assert_almost_equal(res, exp)
 
     def test_get(self):
@@ -366,19 +368,22 @@ def test_get(self):
                            placement=np.arange(3))
         mgr = BlockManager(blocks=[block], axes=[cols, np.arange(3)])
 
-        assert_almost_equal(mgr.get('a'), values[0])
-        assert_almost_equal(mgr.get('b'), values[1])
-        assert_almost_equal(mgr.get('c'), values[2])
+        assert_almost_equal(mgr.get('a', fastpath=False), values[0])
+        assert_almost_equal(mgr.get('b', fastpath=False), values[1])
+        assert_almost_equal(mgr.get('c', fastpath=False), values[2])
+        assert_almost_equal(mgr.get('a').values, values[0])
+        assert_almost_equal(mgr.get('b').values, values[1])
+        assert_almost_equal(mgr.get('c').values, values[2])
 
     def test_set(self):
         mgr = create_mgr('a,b,c: int', item_shape=(3,))
 
         mgr.set('d', np.array(['foo'] * 3))
         mgr.set('b', np.array(['bar'] * 3))
-        assert_almost_equal(mgr.get('a'), [0] * 3)
-        assert_almost_equal(mgr.get('b'), ['bar'] * 3)
-        assert_almost_equal(mgr.get('c'), [2] * 3)
-        assert_almost_equal(mgr.get('d'), ['foo'] * 3)
+        assert_almost_equal(mgr.get('a').values, [0] * 3)
+        assert_almost_equal(mgr.get('b').values, ['bar'] * 3)
+        assert_almost_equal(mgr.get('c').values, [2] * 3)
+        assert_almost_equal(mgr.get('d').values, ['foo'] * 3)
 
     def test_insert(self):
         self.mgr.insert(0, 'inserted', np.arange(N))
@@ -580,10 +585,14 @@ def test_reindex_items(self):
         reindexed = mgr.reindex_axis(['g', 'c', 'a', 'd'], axis=0)
         self.assertEqual(reindexed.nblocks, 2)
         assert_almost_equal(reindexed.items, ['g', 'c', 'a', 'd'])
-        assert_almost_equal(mgr.get('g'), reindexed.get('g'))
-        assert_almost_equal(mgr.get('c'), reindexed.get('c'))
-        assert_almost_equal(mgr.get('a'), reindexed.get('a'))
-        assert_almost_equal(mgr.get('d'), reindexed.get('d'))
+        assert_almost_equal(mgr.get('g',fastpath=False), reindexed.get('g',fastpath=False))
+        assert_almost_equal(mgr.get('c',fastpath=False), reindexed.get('c',fastpath=False))
+        assert_almost_equal(mgr.get('a',fastpath=False), reindexed.get('a',fastpath=False))
+        assert_almost_equal(mgr.get('d',fastpath=False), reindexed.get('d',fastpath=False))
+        assert_almost_equal(mgr.get('g').values, reindexed.get('g').values)
+        assert_almost_equal(mgr.get('c').values, reindexed.get('c').values)
+        assert_almost_equal(mgr.get('a').values, reindexed.get('a').values)
+        assert_almost_equal(mgr.get('d').values, reindexed.get('d').values)
 
     def test_multiindex_xs(self):
         mgr = create_mgr('a,b,c: f8; d,e,f: i8')
@@ -608,16 +617,19 @@ def test_get_numeric_data(self):
 
         numeric = mgr.get_numeric_data()
         assert_almost_equal(numeric.items, ['int', 'float', 'complex', 'bool'])
-        assert_almost_equal(mgr.get('float'), numeric.get('float'))
+        assert_almost_equal(mgr.get('float',fastpath=False), numeric.get('float',fastpath=False))
+        assert_almost_equal(mgr.get('float').values, numeric.get('float').values)
 
         # Check sharing
         numeric.set('float', np.array([100., 200., 300.]))
-        assert_almost_equal(mgr.get('float'), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float',fastpath=False), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float').values, np.array([100., 200., 300.]))
 
         numeric2 = mgr.get_numeric_data(copy=True)
         assert_almost_equal(numeric.items, ['int', 'float', 'complex', 'bool'])
         numeric2.set('float', np.array([1000., 2000., 3000.]))
-        assert_almost_equal(mgr.get('float'), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float',fastpath=False), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float').values, np.array([100., 200., 300.]))
 
     def test_get_bool_data(self):
         mgr = create_mgr('int: int; float: float; complex: complex;'
@@ -627,15 +639,18 @@ def test_get_bool_data(self):
 
         bools = mgr.get_bool_data()
         assert_almost_equal(bools.items, ['bool'])
-        assert_almost_equal(mgr.get('bool'), bools.get('bool'))
+        assert_almost_equal(mgr.get('bool',fastpath=False), bools.get('bool',fastpath=False))
+        assert_almost_equal(mgr.get('bool').values, bools.get('bool').values)
 
         bools.set('bool', np.array([True, False, True]))
-        assert_almost_equal(mgr.get('bool'), [True, False, True])
+        assert_almost_equal(mgr.get('bool',fastpath=False), [True, False, True])
+        assert_almost_equal(mgr.get('bool').values, [True, False, True])
 
         # Check sharing
         bools2 = mgr.get_bool_data(copy=True)
         bools2.set('bool', np.array([False, True, False]))
-        assert_almost_equal(mgr.get('bool'), [True, False, True])
+        assert_almost_equal(mgr.get('bool',fastpath=False), [True, False, True])
+        assert_almost_equal(mgr.get('bool').values, [True, False, True])
 
     def test_unicode_repr_doesnt_raise(self):
         str_repr = repr(create_mgr(u('b,\u05d0: object')))
diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py
--- a/pandas/tests/test_series.py
+++ b/pandas/tests/test_series.py
@@ -581,6 +581,12 @@ def test_constructor_pass_none(self):
         s = Series(None, index=lrange(5), dtype=object)
         self.assertEqual(s.dtype, np.object_)
 
+        # GH 7431
+        # inference on the index
+        s = Series(index=np.array([None]))
+        expected = Series(index=Index([None]))
+        assert_series_equal(s,expected)
+
     def test_constructor_cast(self):
         self.assertRaises(ValueError, Series, ['a', 'b', 'c'], dtype=float)
 
@@ -669,6 +675,16 @@ def test_constructor_dtype_datetime64(self):
         self.assert_numpy_array_equal(series1.values,dates2)
         self.assertEqual(series1.dtype,object)
 
+        # these will correctly infer a datetime
+        s = Series([None, pd.NaT, '2013-08-05 15:30:00.000001'])
+        self.assertEqual(s.dtype,'datetime64[ns]')
+        s = Series([np.nan, pd.NaT, '2013-08-05 15:30:00.000001'])
+        self.assertEqual(s.dtype,'datetime64[ns]')
+        s = Series([pd.NaT, None, '2013-08-05 15:30:00.000001'])
+        self.assertEqual(s.dtype,'datetime64[ns]')
+        s = Series([pd.NaT, np.nan, '2013-08-05 15:30:00.000001'])
+        self.assertEqual(s.dtype,'datetime64[ns]')
+
     def test_constructor_dict(self):
         d = {'a': 0., 'b': 1., 'c': 2.}
         result = Series(d, index=['b', 'c', 'd', 'a'])
@@ -2462,6 +2478,18 @@ def f():
         td = Series([timedelta(days=i) for i in range(3)] + ['foo'])
         self.assertEqual(td.dtype, 'object')
 
+        # these will correctly infer a timedelta
+        # but only on numpy > 1.7 as the cython path will only be used
+        if not _np_version_under1p7:
+            s = Series([None, pd.NaT, '1 Day'])
+            self.assertEqual(s.dtype,'timedelta64[ns]')
+            s = Series([np.nan, pd.NaT, '1 Day'])
+            self.assertEqual(s.dtype,'timedelta64[ns]')
+            s = Series([pd.NaT, None, '1 Day'])
+            self.assertEqual(s.dtype,'timedelta64[ns]')
+            s = Series([pd.NaT, np.nan, '1 Day'])
+            self.assertEqual(s.dtype,'timedelta64[ns]')
+
     def test_operators_timedelta64(self):
 
         # invalid ops
@@ -2939,12 +2967,12 @@ def test_datetime64_fillna(self):
 
         # GH 6587
         # make sure that we are treating as integer when filling
+        # this also tests inference of a datetime-like with NaT's
         s = Series([pd.NaT, pd.NaT, '2013-08-05 15:30:00.000001'])
         expected = Series(['2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001'], dtype='M8[ns]')
         result = s.fillna(method='backfill')
         assert_series_equal(result, expected)
 
-
     def test_fillna_int(self):
         s = Series(np.random.randint(-100, 100, 50))
         s.fillna(method='ffill', inplace=True)
diff --git a/pandas/tests/test_tseries.py b/pandas/tests/test_tseries.py
--- a/pandas/tests/test_tseries.py
+++ b/pandas/tests/test_tseries.py
@@ -658,6 +658,13 @@ def test_to_object_array_tuples(self):
         except ImportError:
             pass
 
+    def test_object(self):
+
+        # GH 7431
+        # cannot infer more than this as only a single element
+        arr = np.array([None],dtype='O')
+        result = lib.infer_dtype(arr)
+        self.assertEqual(result, 'mixed')
 
 class TestMoments(tm.TestCase):
     pass
diff --git a/pandas/tseries/tests/test_timeseries.py b/pandas/tseries/tests/test_timeseries.py
--- a/pandas/tseries/tests/test_timeseries.py
+++ b/pandas/tseries/tests/test_timeseries.py
@@ -58,7 +58,7 @@ def _skip_if_has_locale():
     lang, _ = locale.getlocale()
     if lang is not None:
         raise nose.SkipTest("Specific locale is set {0}".format(lang))
-    
+
 def _skip_if_windows_python_3():
     if sys.version_info > (3,) and sys.platform == 'win32':
         raise nose.SkipTest("not used on python 3/win32")