Repository: pandas-dev/pandas

Issue:
series with datetime raises pandas.tslib.OutOfBoundsDatetime on value_counts()
I expected that for each value a the number of entries would be returned, instead it throws an OutOfBoundsDatetime although it is not pandas object in first place, but a datetime object. I am aware that the pandas timestamps have a limited time span, however the objects are not timestamps, but datetimes... Thanks!
#### Code Sample, a copy-pastable example if possible

```
import pandas as pd
import datetime
s = pd.Series([datetime.datetime(9999,1,1)])
s.value_counts()
```
#### Expected Output

```
---------------------------------------------------------------------------
OutOfBoundsDatetime                       Traceback (most recent call last)
<ipython-input-7-1b9293e6eb60> in <module>()
----> 1 s.value_counts()

/usr/lib/python2.7/dist-packages/pandas/core/base.pyc in value_counts(self, normalize, sort, ascending, bins, dropna)
    466         from pandas.tseries.api import DatetimeIndex, PeriodIndex
    467         result = value_counts(self, sort=sort, ascending=ascending,
--> 468                               normalize=normalize, bins=bins, dropna=dropna)
    469 
    470         if isinstance(self, PeriodIndex):

/usr/lib/python2.7/dist-packages/pandas/core/algorithms.pyc in value_counts(values, sort, ascending, normalize, bins, dropna)
    318 
    319         if not isinstance(keys, Index):
--> 320             keys = Index(keys)
    321         result = Series(counts, index=keys, name=name)
    322 

/usr/lib/python2.7/dist-packages/pandas/core/index.pyc in __new__(cls, data, dtype, copy, name, fastpath, tupleize_cols, **kwargs)
    177                         tslib.is_timestamp_array(subarr)):
    178                         from pandas.tseries.index import DatetimeIndex
--> 179                         return DatetimeIndex(subarr, copy=copy, name=name, **kwargs)
    180                     elif (inferred.startswith('timedelta') or
    181                           lib.is_timedelta_array(subarr)):

/usr/lib/python2.7/dist-packages/pandas/util/decorators.pyc in wrapper(*args, **kwargs)
     87                 else:
     88                     kwargs[new_arg_name] = new_arg_value
---> 89             return func(*args, **kwargs)
     90         return wrapper
     91     return _deprecate_kwarg

/usr/lib/python2.7/dist-packages/pandas/tseries/index.pyc in __new__(cls, data, freq, start, end, periods, copy, name, tz, verify_integrity, normalize, closed, ambiguous, dtype, **kwargs)
    317                 except ValueError:
    318                     # tz aware
--> 319                     subarr = tools._to_datetime(data, box=False, utc=True)
    320 
    321                 # we may not have been able to convert

/usr/lib/python2.7/dist-packages/pandas/tseries/tools.pyc in _to_datetime(arg, errors, dayfirst, yearfirst, utc, box, format, exact, unit, freq, infer_datetime_format)
    393         return _convert_listlike(arg, box, format, name=arg.name)
    394     elif com.is_list_like(arg):
--> 395         return _convert_listlike(arg, box, format)
    396 
    397     return _convert_listlike(np.array([ arg ]), box, format)[0]

/usr/lib/python2.7/dist-packages/pandas/tseries/tools.pyc in _convert_listlike(arg, box, format, name)
    381                 return DatetimeIndex._simple_new(values, name=name, tz=tz)
    382             except (ValueError, TypeError):
--> 383                 raise e
    384 
    385     if arg is None:

OutOfBoundsDatetime: Out of bounds nanosecond timestamp: 9999-01-01 00:00:00
```
#### output of `pd.show_versions()`

```
INSTALLED VERSIONS
------------------
commit: None
python: 2.7.11.final.0
python-bits: 64
OS: Linux
OS-release: 4.4.0-28-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: de_DE.UTF-8

pandas: 0.17.1
nose: 1.3.7
pip: 8.1.1
setuptools: 20.7.0
Cython: None
numpy: 1.11.0
scipy: 0.17.0
statsmodels: 0.6.1
IPython: 2.4.1
sphinx: None
patsy: 0.4.1
dateutil: 2.4.2
pytz: 2014.10
blosc: None
bottleneck: None
tables: 3.2.2
numexpr: 2.4.3
matplotlib: 1.5.1
openpyxl: 2.3.0
xlrd: 0.9.4
xlwt: 0.7.5
xlsxwriter: None
lxml: 3.5.0
bs4: 4.4.1
html5lib: 0.999
httplib2: 0.9.1
apiclient: None
sqlalchemy: 1.0.11
pymysql: None
psycopg2: None
Jinja2: None


```



Discussion:
It looks like the problem comes when we try to construct the index [here](https://github.com/pydata/pandas/blob/d7c028d4965932160fa3b69f56c716b1454c42a5/pandas/core/algorithms.py#L303). Essentially we call `Index([datetime.datetime(9999,1,1)])` instead of `Index([datetime.datetime(9999,1,1)], dtype=object)`. Want to see if making that change fixes it without breaking any tests?

Any reason you're using `datetime.datetime`s with object dtype instead of `Period`s?

``` python
In [4]: pd.Series([pd.Period('9999-01-01')]).value_counts()
Out[4]:
9999-01-01    1
Freq: D, dtype: int64
```

That might be less painful for you since there will surely be other places in pandas that don't support datetimes with object dtype. It might not be worth fixing this one case, if Periods provide a better alternative.

I read the data from a sql-database via `pd.read_sql_query` and for values that do not go outside the boundaries the type is `datetime64[ns]`, however infinity is marked as 9999 in some columns and the `dtype` for those is `object`.
Thanks for the advice on periods.


Resolving patch:
diff --git a/pandas/core/series.py b/pandas/core/series.py
--- a/pandas/core/series.py
+++ b/pandas/core/series.py
@@ -291,11 +291,13 @@ def _set_axis(self, axis, labels, fastpath=False):
 
             if not isinstance(labels,
                               (DatetimeIndex, PeriodIndex, TimedeltaIndex)):
-                labels = DatetimeIndex(labels)
-
-                # need to set here becuase we changed the index
-                if fastpath:
-                    self._data.set_axis(axis, labels)
+                try:
+                    labels = DatetimeIndex(labels)
+                    # need to set here becuase we changed the index
+                    if fastpath:
+                        self._data.set_axis(axis, labels)
+                except tslib.OutOfBoundsDatetime:
+                    pass
         self._set_subtyp(is_all_dates)
 
         object.__setattr__(self, '_index', labels)
diff --git a/pandas/indexes/base.py b/pandas/indexes/base.py
--- a/pandas/indexes/base.py
+++ b/pandas/indexes/base.py
@@ -258,13 +258,15 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                     pass
                 elif inferred != 'string':
                     if inferred.startswith('datetime'):
-
                         if (lib.is_datetime_with_singletz_array(subarr) or
                                 'tz' in kwargs):
                             # only when subarr has the same tz
                             from pandas.tseries.index import DatetimeIndex
-                            return DatetimeIndex(subarr, copy=copy, name=name,
-                                                 **kwargs)
+                            try:
+                                return DatetimeIndex(subarr, copy=copy,
+                                                     name=name, **kwargs)
+                            except tslib.OutOfBoundsDatetime:
+                                pass
 
                     elif inferred.startswith('timedelta'):
                         from pandas.tseries.tdi import TimedeltaIndex


Test patch:
diff --git a/pandas/tests/indexes/test_datetimelike.py b/pandas/tests/indexes/test_datetimelike.py
--- a/pandas/tests/indexes/test_datetimelike.py
+++ b/pandas/tests/indexes/test_datetimelike.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from datetime import timedelta, time
+from datetime import datetime, timedelta, time
 
 import numpy as np
 
@@ -12,7 +12,7 @@
 import pandas.util.testing as tm
 
 import pandas as pd
-from pandas.lib import Timestamp
+from pandas.tslib import Timestamp, OutOfBoundsDatetime
 
 from .common import Base
 
@@ -336,6 +336,18 @@ def test_construction_base_constructor(self):
         tm.assert_index_equal(pd.Index(np.array(arr)),
                               pd.DatetimeIndex(np.array(arr)))
 
+    def test_construction_outofbounds(self):
+        # GH 13663
+        dates = [datetime(3000, 1, 1), datetime(4000, 1, 1),
+                 datetime(5000, 1, 1), datetime(6000, 1, 1)]
+        exp = Index(dates, dtype=object)
+        # coerces to object
+        tm.assert_index_equal(Index(dates), exp)
+
+        with tm.assertRaises(OutOfBoundsDatetime):
+            # can't create DatetimeIndex
+            DatetimeIndex(dates)
+
     def test_astype(self):
         # GH 13149, GH 13209
         idx = DatetimeIndex(['2016-05-16', 'NaT', NaT, np.NaN])
diff --git a/pandas/tests/test_algos.py b/pandas/tests/test_algos.py
--- a/pandas/tests/test_algos.py
+++ b/pandas/tests/test_algos.py
@@ -4,7 +4,7 @@
 import numpy as np
 from numpy.random import RandomState
 from numpy import nan
-import datetime
+from datetime import datetime
 from pandas import Series, Categorical, CategoricalIndex, Index
 import pandas as pd
 
@@ -121,7 +121,7 @@ def test_mixed_integer(self):
 
     def test_unsortable(self):
         # GH 13714
-        arr = np.array([1, 2, datetime.datetime.now(), 0, 3], dtype=object)
+        arr = np.array([1, 2, datetime.now(), 0, 3], dtype=object)
         if compat.PY2 and not pd._np_version_under1p10:
             # RuntimeWarning: tp_compare didn't return -1 or -2 for exception
             with tm.assert_produces_warning(RuntimeWarning):
@@ -556,6 +556,18 @@ def test_value_counts_nat(self):
         tm.assert_series_equal(algos.value_counts(dt), exp_dt)
         # TODO same for (timedelta)
 
+    def test_value_counts_datetime_outofbounds(self):
+        # GH 13663
+        s = pd.Series([datetime(3000, 1, 1), datetime(5000, 1, 1),
+                       datetime(5000, 1, 1), datetime(6000, 1, 1),
+                       datetime(3000, 1, 1), datetime(3000, 1, 1)])
+        res = s.value_counts()
+
+        exp_index = pd.Index([datetime(3000, 1, 1), datetime(5000, 1, 1),
+                              datetime(6000, 1, 1)], dtype=object)
+        exp = pd.Series([3, 2, 1], index=exp_index)
+        tm.assert_series_equal(res, exp)
+
     def test_categorical(self):
         s = Series(pd.Categorical(list('aaabbc')))
         result = s.value_counts()
@@ -818,7 +830,7 @@ def _check(arr):
 def test_pad_backfill_object_segfault():
 
     old = np.array([], dtype='O')
-    new = np.array([datetime.datetime(2010, 12, 31)], dtype='O')
+    new = np.array([datetime(2010, 12, 31)], dtype='O')
 
     result = _algos.pad_object(old, new)
     expected = np.array([-1], dtype=np.int64)