Repository: pandas-dev/pandas

Issue:
DataFrame.from_records doesn't handle missing dates (None)
When you construct a DataFrame from a numpy recarray with datetime data and None for missing dates you get an error.

``` python
arrdata = [np.array([datetime.datetime(2005, 3, 1, 0, 0), None])]
dtypes = [('EXPIRY', '<M8[m]')]
recarray = np.core.records.fromarrays(arrdata, dtype=dtypes)

df = pd.DataFrame.from_records(recarray)
Traceback (most recent call last):
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/ipython-1.1.0_1_ahl1-py2.7.egg/IPython/core/interactiveshell.py", line 2830, in run_code
    exec code_obj in self.user_global_ns, self.user_ns
  File "<ipython-input-33-ae01f48c3b82>", line 1, in <module>
    df = pd.DataFrame.from_records(recarray)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_408_g464c1f9-py2.7-linux-x86_64.egg/pandas/core/frame.py", line 841, in from_records
    columns)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_408_g464c1f9-py2.7-linux-x86_64.egg/pandas/core/frame.py", line 4473, in _arrays_to_mgr
    return create_block_manager_from_arrays(arrays, arr_names, axes)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_408_g464c1f9-py2.7-linux-x86_64.egg/pandas/core/internals.py", line 3748, in create_block_manager_from_arrays
    construction_error(len(arrays), arrays[0].shape[1:], axes, e)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_408_g464c1f9-py2.7-linux-x86_64.egg/pandas/core/internals.py", line 3720, in construction_error
    passed,implied))
ValueError: Shape of passed values is (1,), indices imply (1, 2)
```

This is a regression from 0.11.0. Stepping through the code it looks the initial error is raised in tslib.cast_to_nanoseconds and then caught and re-raised in create_block_manager_from_arrays

Incidentally, construction does work from a simple array instead of a recarray:

``` python
pd.DataFrame(np.array([datetime.datetime(2005, 3, 1, 0, 0), None]))
Out[36]: 
           0
0 2005-03-01
1        NaT

[2 rows x 1 columns]
```



Discussion:
hmm prob not testing this case
on master too?

Yes, this is on master.


Resolving patch:
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -41,9 +41,7 @@ class AmbiguousIndexError(PandasError, KeyError):
 
 
 _POSSIBLY_CAST_DTYPES = set([np.dtype(t).name
-                             for t in ['M8[ns]', '>M8[ns]', '<M8[ns]',
-                                       'm8[ns]', '>m8[ns]', '<m8[ns]',
-                                       'O', 'int8',
+                             for t in ['O', 'int8',
                                        'uint8', 'int16', 'uint16', 'int32',
                                        'uint32', 'int64', 'uint64']])
 
@@ -1612,6 +1610,14 @@ def _possibly_convert_objects(values, convert_dates=True,
 
 
 def _possibly_castable(arr):
+    # return False to force a non-fastpath
+
+    # check datetime64[ns]/timedelta64[ns] are valid
+    # otherwise try to coerce
+    kind = arr.dtype.kind
+    if kind == 'M' or kind == 'm':
+        return arr.dtype in _DATELIKE_DTYPES
+
     return arr.dtype.name not in _POSSIBLY_CAST_DTYPES
 
 
@@ -1681,12 +1687,30 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
 
     else:
 
+        is_array = isinstance(value, np.ndarray)
+
+        # catch a datetime/timedelta that is not of ns variety
+        # and no coercion specified
+        if (is_array and value.dtype.kind in ['M','m']):
+            dtype = value.dtype
+
+            if dtype.kind == 'M' and dtype != _NS_DTYPE:
+                try:
+                    value = tslib.array_to_datetime(value)
+                except:
+                    raise
+
+            elif dtype.kind == 'm' and dtype != _TD_DTYPE:
+                from pandas.tseries.timedeltas import \
+                     _possibly_cast_to_timedelta
+                value = _possibly_cast_to_timedelta(value, coerce='compat')
+
         # only do this if we have an array and the dtype of the array is not
         # setup already we are not an integer/object, so don't bother with this
         # conversion
-        if (isinstance(value, np.ndarray) and not
-                (issubclass(value.dtype.type, np.integer) or
-                 value.dtype == np.object_)):
+        elif (is_array and not (
+            issubclass(value.dtype.type, np.integer) or
+            value.dtype == np.object_)):
             pass
 
         else:
diff --git a/pandas/core/series.py b/pandas/core/series.py
--- a/pandas/core/series.py
+++ b/pandas/core/series.py
@@ -2536,7 +2536,10 @@ def _try_cast(arr, take_fast_path):
                 else:
                     subarr = _try_cast(data, True)
         else:
-            subarr = _try_cast(data, True)
+            # don't coerce Index types
+            # e.g. indexes can have different conversions (so don't fast path them)
+            # GH 6140
+            subarr = _try_cast(data, not isinstance(data, Index))
 
         if copy:
             subarr = data.copy()


Test patch:
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -4,6 +4,7 @@
 # pylint: disable-msg=W0612,E1101
 from copy import deepcopy
 from datetime import datetime, timedelta, time
+import sys
 import operator
 import re
 import csv
@@ -3956,6 +3957,27 @@ def test_from_records_empty_with_nonempty_fields_gh3682(self):
         assert_array_equal(df.index, Index([], name='id'))
         self.assertEqual(df.index.name, 'id')
 
+    def test_from_records_with_datetimes(self):
+        if sys.version < LooseVersion('2.7'):
+            raise nose.SkipTest('rec arrays dont work properly with py2.6')
+
+        # construction with a null in a recarray
+        # GH 6140
+        expected = DataFrame({ 'EXPIRY'  : [datetime(2005, 3, 1, 0, 0), None ]})
+
+        arrdata = [np.array([datetime(2005, 3, 1, 0, 0), None])]
+        dtypes = [('EXPIRY', '<M8[ns]')]
+        recarray = np.core.records.fromarrays(arrdata, dtype=dtypes)
+        result = DataFrame.from_records(recarray)
+        assert_frame_equal(result,expected)
+
+        # coercion should work too
+        arrdata = [np.array([datetime(2005, 3, 1, 0, 0), None])]
+        dtypes = [('EXPIRY', '<M8[m]')]
+        recarray = np.core.records.fromarrays(arrdata, dtype=dtypes)
+        result = DataFrame.from_records(recarray)
+        assert_frame_equal(result,expected)
+
     def test_to_records_floats(self):
         df = DataFrame(np.random.rand(10, 10))
         df.to_records()
@@ -5138,8 +5160,6 @@ def test_combineSeries(self):
         #_check_mixed_int(added, dtype = dict(A = 'int32', B = 'float64', C = 'int32', D = 'int64'))
 
         # TimeSeries
-        import sys
-
         buf = StringIO()
         tmp = sys.stderr
         sys.stderr = buf